PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  30 use PVE::DataCenterConfig;
  31 use PVE::Exception qw(raise raise_param_exc);
  32 use PVE::GuestHelpers;
  33 use PVE::INotify;
  34 use PVE::JSONSchema qw(get_standard_option);
  35 use PVE::ProcFSTools;
  36 use PVE::RPCEnvironment;
  37 use PVE::Storage;
  38 use PVE::SysFSTools;
  39 use PVE::Systemd;
  40 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  41
  42 use PVE::QMPClient;
  43 use PVE::QemuConfig;
  44 use PVE::QemuServer::Helpers qw(min_version);
  45 use PVE::QemuServer::Cloudinit;
  46 use PVE::QemuServer::Machine;
  47 use PVE::QemuServer::Memory;
  48 use PVE::QemuServer::Monitor qw(mon_cmd);
  49 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port);
  50 use PVE::QemuServer::USB qw(parse_usb_device);
  51
  52 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  53 my $OVMF = {
  54     x86_64 => [
  55         "$EDK2_FW_BASE/OVMF_CODE.fd",
  56         "$EDK2_FW_BASE/OVMF_VARS.fd"
  57     ],
  58     aarch64 => [
  59         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  60         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  61     ],
  62 };
  63
  64 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  65
  66 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  67
  68 # Note about locking: we use flock on the config file protect
  69 # against concurent actions.
  70 # Aditionaly, we have a 'lock' setting in the config file. This
  71 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  72 # allowed when such lock is set. But you can ignore this kind of
  73 # lock with the --skiplock flag.
  74
  75 cfs_register_file('/qemu-server/',
  76                   \&parse_vm_config,
  77                   \&write_vm_config);
  78
  79 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  80     description => "Some command save/restore state from this location.",
  81     type => 'string',
  82     maxLength => 128,
  83     optional => 1,
  84 });
  85
  86 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  87     type => 'string',
  88     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  89     description => "The drive's backing file's data format.",
  90     optional => 1,
  91 });
  92
  93 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  94         description => "Specifies the Qemu machine type.",
  95         type => 'string',
  96         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
  97         maxLength => 40,
  98         optional => 1,
  99 });
 100
 101 #no warnings 'redefine';
 102
 103 sub cgroups_write {
 104    my ($controller, $vmid, $option, $value) = @_;
 105
 106    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 107    PVE::ProcFSTools::write_proc_entry($path, $value);
 108
 109 }
 110
 111 my $nodename_cache;
 112 sub nodename {
 113     $nodename_cache //= PVE::INotify::nodename();
 114     return $nodename_cache;
 115 }
 116
 117 my $cpu_vendor_list = {
 118     # Intel CPUs
 119     486 => 'GenuineIntel',
 120     pentium => 'GenuineIntel',
 121     pentium2  => 'GenuineIntel',
 122     pentium3  => 'GenuineIntel',
 123     coreduo => 'GenuineIntel',
 124     core2duo => 'GenuineIntel',
 125     Conroe  => 'GenuineIntel',
 126     Penryn  => 'GenuineIntel',
 127     Nehalem  => 'GenuineIntel',
 128     'Nehalem-IBRS'  => 'GenuineIntel',
 129     Westmere => 'GenuineIntel',
 130     'Westmere-IBRS' => 'GenuineIntel',
 131     SandyBridge => 'GenuineIntel',
 132     'SandyBridge-IBRS' => 'GenuineIntel',
 133     IvyBridge => 'GenuineIntel',
 134     'IvyBridge-IBRS' => 'GenuineIntel',
 135     Haswell => 'GenuineIntel',
 136     'Haswell-IBRS' => 'GenuineIntel',
 137     'Haswell-noTSX' => 'GenuineIntel',
 138     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 139     Broadwell => 'GenuineIntel',
 140     'Broadwell-IBRS' => 'GenuineIntel',
 141     'Broadwell-noTSX' => 'GenuineIntel',
 142     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 143     'Skylake-Client' => 'GenuineIntel',
 144     'Skylake-Client-IBRS' => 'GenuineIntel',
 145     'Skylake-Server' => 'GenuineIntel',
 146     'Skylake-Server-IBRS' => 'GenuineIntel',
 147     'Cascadelake-Server' => 'GenuineIntel',
 148     KnightsMill => 'GenuineIntel',
 149
 150
 151     # AMD CPUs
 152     athlon => 'AuthenticAMD',
 153     phenom  => 'AuthenticAMD',
 154     Opteron_G1  => 'AuthenticAMD',
 155     Opteron_G2  => 'AuthenticAMD',
 156     Opteron_G3  => 'AuthenticAMD',
 157     Opteron_G4  => 'AuthenticAMD',
 158     Opteron_G5  => 'AuthenticAMD',
 159     EPYC => 'AuthenticAMD',
 160     'EPYC-IBPB' => 'AuthenticAMD',
 161
 162     # generic types, use vendor from host node
 163     host => 'default',
 164     kvm32 => 'default',
 165     kvm64 => 'default',
 166     qemu32 => 'default',
 167     qemu64 => 'default',
 168     max => 'default',
 169 };
 170
 171 my @supported_cpu_flags = (
 172     'pcid',
 173     'spec-ctrl',
 174     'ibpb',
 175     'ssbd',
 176     'virt-ssbd',
 177     'amd-ssbd',
 178     'amd-no-ssb',
 179     'pdpe1gb',
 180     'md-clear',
 181     'hv-tlbflush',
 182     'hv-evmcs',
 183     'aes'
 184 );
 185 my $cpu_flag = qr/[+-](@{[join('|', @supported_cpu_flags)]})/;
 186
 187 my $cpu_fmt = {
 188     cputype => {
 189         description => "Emulated CPU type.",
 190         type => 'string',
 191         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 192         default => 'kvm64',
 193         default_key => 1,
 194     },
 195     hidden => {
 196         description => "Do not identify as a KVM virtual machine.",
 197         type => 'boolean',
 198         optional => 1,
 199         default => 0
 200     },
 201     'hv-vendor-id' => {
 202         type => 'string',
 203         pattern => qr/[a-zA-Z0-9]{1,12}/,
 204         format_description => 'vendor-id',
 205         description => 'The Hyper-V vendor ID. Some drivers or programs inside Windows guests need a specific ID.',
 206         optional => 1,
 207     },
 208     flags => {
 209         description => "List of additional CPU flags separated by ';'."
 210                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 211                      . " Currently supported flags: @{[join(', ', @supported_cpu_flags)]}.",
 212         format_description => '+FLAG[;-FLAG...]',
 213         type => 'string',
 214         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 215         optional => 1,
 216     },
 217 };
 218
 219 my $watchdog_fmt = {
 220     model => {
 221         default_key => 1,
 222         type => 'string',
 223         enum => [qw(i6300esb ib700)],
 224         description => "Watchdog type to emulate.",
 225         default => 'i6300esb',
 226         optional => 1,
 227     },
 228     action => {
 229         type => 'string',
 230         enum => [qw(reset shutdown poweroff pause debug none)],
 231         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 232         optional => 1,
 233     },
 234 };
 235 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 236
 237 my $agent_fmt = {
 238     enabled => {
 239         description => "Enable/disable Qemu GuestAgent.",
 240         type => 'boolean',
 241         default => 0,
 242         default_key => 1,
 243     },
 244     fstrim_cloned_disks => {
 245         description => "Run fstrim after cloning/moving a disk.",
 246         type => 'boolean',
 247         optional => 1,
 248         default => 0
 249     },
 250     type => {
 251         description => "Select the agent type",
 252         type => 'string',
 253         default => 'virtio',
 254         optional => 1,
 255         enum => [qw(virtio isa)],
 256     },
 257 };
 258
 259 my $vga_fmt = {
 260     type => {
 261         description => "Select the VGA type.",
 262         type => 'string',
 263         default => 'std',
 264         optional => 1,
 265         default_key => 1,
 266         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 267     },
 268     memory => {
 269         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 270         type => 'integer',
 271         optional => 1,
 272         minimum => 4,
 273         maximum => 512,
 274     },
 275 };
 276
 277 my $ivshmem_fmt = {
 278     size => {
 279         type => 'integer',
 280         minimum => 1,
 281         description => "The size of the file in MB.",
 282     },
 283     name => {
 284         type => 'string',
 285         pattern => '[a-zA-Z0-9\-]+',
 286         optional => 1,
 287         format_description => 'string',
 288         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 289     },
 290 };
 291
 292 my $audio_fmt = {
 293     device => {
 294         type => 'string',
 295         enum => [qw(ich9-intel-hda intel-hda AC97)],
 296         description =>  "Configure an audio device."
 297     },
 298     driver =>  {
 299         type => 'string',
 300         enum => ['spice'],
 301         default => 'spice',
 302         optional => 1,
 303         description => "Driver backend for the audio device."
 304     },
 305 };
 306
 307 my $spice_enhancements_fmt = {
 308     foldersharing => {
 309         type => 'boolean',
 310         optional => 1,
 311         default => '0',
 312         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 313     },
 314     videostreaming =>  {
 315         type => 'string',
 316         enum => ['off', 'all', 'filter'],
 317         default => 'off',
 318         optional => 1,
 319         description => "Enable video streaming. Uses compression for detected video streams."
 320     },
 321 };
 322
 323 my $confdesc = {
 324     onboot => {
 325         optional => 1,
 326         type => 'boolean',
 327         description => "Specifies whether a VM will be started during system bootup.",
 328         default => 0,
 329     },
 330     autostart => {
 331         optional => 1,
 332         type => 'boolean',
 333         description => "Automatic restart after crash (currently ignored).",
 334         default => 0,
 335     },
 336     hotplug => {
 337         optional => 1,
 338         type => 'string', format => 'pve-hotplug-features',
 339         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 340         default => 'network,disk,usb',
 341     },
 342     reboot => {
 343         optional => 1,
 344         type => 'boolean',
 345         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 346         default => 1,
 347     },
 348     lock => {
 349         optional => 1,
 350         type => 'string',
 351         description => "Lock/unlock the VM.",
 352         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 353     },
 354     cpulimit => {
 355         optional => 1,
 356         type => 'number',
 357         description => "Limit of CPU usage.",
 358         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 359         minimum => 0,
 360         maximum => 128,
 361         default => 0,
 362     },
 363     cpuunits => {
 364         optional => 1,
 365         type => 'integer',
 366         description => "CPU weight for a VM.",
 367         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 368         minimum => 2,
 369         maximum => 262144,
 370         default => 1024,
 371     },
 372     memory => {
 373         optional => 1,
 374         type => 'integer',
 375         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 376         minimum => 16,
 377         default => 512,
 378     },
 379     balloon => {
 380         optional => 1,
 381         type => 'integer',
 382         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 383         minimum => 0,
 384     },
 385     shares => {
 386         optional => 1,
 387         type => 'integer',
 388         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 389         minimum => 0,
 390         maximum => 50000,
 391         default => 1000,
 392     },
 393     keyboard => {
 394         optional => 1,
 395         type => 'string',
 396         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 397                        "It should not be necessary to set it.",
 398         enum => PVE::Tools::kvmkeymaplist(),
 399         default => undef,
 400     },
 401     name => {
 402         optional => 1,
 403         type => 'string', format => 'dns-name',
 404         description => "Set a name for the VM. Only used on the configuration web interface.",
 405     },
 406     scsihw => {
 407         optional => 1,
 408         type => 'string',
 409         description => "SCSI controller model",
 410         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 411         default => 'lsi',
 412     },
 413     description => {
 414         optional => 1,
 415         type => 'string',
 416         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 417     },
 418     ostype => {
 419         optional => 1,
 420         type => 'string',
 421         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 422         description => "Specify guest operating system.",
 423         verbose_description => <<EODESC,
 424 Specify guest operating system. This is used to enable special
 425 optimization/features for specific operating systems:
 426
 427 [horizontal]
 428 other;; unspecified OS
 429 wxp;; Microsoft Windows XP
 430 w2k;; Microsoft Windows 2000
 431 w2k3;; Microsoft Windows 2003
 432 w2k8;; Microsoft Windows 2008
 433 wvista;; Microsoft Windows Vista
 434 win7;; Microsoft Windows 7
 435 win8;; Microsoft Windows 8/2012/2012r2
 436 win10;; Microsoft Windows 10/2016
 437 l24;; Linux 2.4 Kernel
 438 l26;; Linux 2.6 - 5.X Kernel
 439 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 440 EODESC
 441     },
 442     boot => {
 443         optional => 1,
 444         type => 'string',
 445         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 446         pattern => '[acdn]{1,4}',
 447         default => 'cdn',
 448     },
 449     bootdisk => {
 450         optional => 1,
 451         type => 'string', format => 'pve-qm-bootdisk',
 452         description => "Enable booting from specified disk.",
 453         pattern => '(ide|sata|scsi|virtio)\d+',
 454     },
 455     smp => {
 456         optional => 1,
 457         type => 'integer',
 458         description => "The number of CPUs. Please use option -sockets instead.",
 459         minimum => 1,
 460         default => 1,
 461     },
 462     sockets => {
 463         optional => 1,
 464         type => 'integer',
 465         description => "The number of CPU sockets.",
 466         minimum => 1,
 467         default => 1,
 468     },
 469     cores => {
 470         optional => 1,
 471         type => 'integer',
 472         description => "The number of cores per socket.",
 473         minimum => 1,
 474         default => 1,
 475     },
 476     numa => {
 477         optional => 1,
 478         type => 'boolean',
 479         description => "Enable/disable NUMA.",
 480         default => 0,
 481     },
 482     hugepages => {
 483         optional => 1,
 484         type => 'string',
 485         description => "Enable/disable hugepages memory.",
 486         enum => [qw(any 2 1024)],
 487     },
 488     vcpus => {
 489         optional => 1,
 490         type => 'integer',
 491         description => "Number of hotplugged vcpus.",
 492         minimum => 1,
 493         default => 0,
 494     },
 495     acpi => {
 496         optional => 1,
 497         type => 'boolean',
 498         description => "Enable/disable ACPI.",
 499         default => 1,
 500     },
 501     agent => {
 502         optional => 1,
 503         description => "Enable/disable Qemu GuestAgent and its properties.",
 504         type => 'string',
 505         format => $agent_fmt,
 506     },
 507     kvm => {
 508         optional => 1,
 509         type => 'boolean',
 510         description => "Enable/disable KVM hardware virtualization.",
 511         default => 1,
 512     },
 513     tdf => {
 514         optional => 1,
 515         type => 'boolean',
 516         description => "Enable/disable time drift fix.",
 517         default => 0,
 518     },
 519     localtime => {
 520         optional => 1,
 521         type => 'boolean',
 522         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 523     },
 524     freeze => {
 525         optional => 1,
 526         type => 'boolean',
 527         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 528     },
 529     vga => {
 530         optional => 1,
 531         type => 'string', format => $vga_fmt,
 532         description => "Configure the VGA hardware.",
 533         verbose_description => "Configure the VGA Hardware. If you want to use ".
 534             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 535             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 536             "is 'std' for all OS types besides some Windows versions (XP and " .
 537             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 538             "display server. For win* OS you can select how many independent " .
 539             "displays you want, Linux guests can add displays them self.\n".
 540             "You can also run without any graphic card, using a serial device as terminal.",
 541     },
 542     watchdog => {
 543         optional => 1,
 544         type => 'string', format => 'pve-qm-watchdog',
 545         description => "Create a virtual hardware watchdog device.",
 546         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 547             " (by a guest action), the watchdog must be periodically polled " .
 548             "by an agent inside the guest or else the watchdog will reset " .
 549             "the guest (or execute the respective action specified)",
 550     },
 551     startdate => {
 552         optional => 1,
 553         type => 'string',
 554         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 555         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 556         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 557         default => 'now',
 558     },
 559     startup =>  get_standard_option('pve-startup-order'),
 560     template => {
 561         optional => 1,
 562         type => 'boolean',
 563         description => "Enable/disable Template.",
 564         default => 0,
 565     },
 566     args => {
 567         optional => 1,
 568         type => 'string',
 569         description => "Arbitrary arguments passed to kvm.",
 570         verbose_description => <<EODESCR,
 571 Arbitrary arguments passed to kvm, for example:
 572
 573 args: -no-reboot -no-hpet
 574
 575 NOTE: this option is for experts only.
 576 EODESCR
 577     },
 578     tablet => {
 579         optional => 1,
 580         type => 'boolean',
 581         default => 1,
 582         description => "Enable/disable the USB tablet device.",
 583         verbose_description => "Enable/disable the USB tablet device. This device is " .
 584             "usually needed to allow absolute mouse positioning with VNC. " .
 585             "Else the mouse runs out of sync with normal VNC clients. " .
 586             "If you're running lots of console-only guests on one host, " .
 587             "you may consider disabling this to save some context switches. " .
 588             "This is turned off by default if you use spice (-vga=qxl).",
 589     },
 590     migrate_speed => {
 591         optional => 1,
 592         type => 'integer',
 593         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 594         minimum => 0,
 595         default => 0,
 596     },
 597     migrate_downtime => {
 598         optional => 1,
 599         type => 'number',
 600         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 601         minimum => 0,
 602         default => 0.1,
 603     },
 604     cdrom => {
 605         optional => 1,
 606         type => 'string', format => 'pve-qm-ide',
 607         typetext => '<volume>',
 608         description => "This is an alias for option -ide2",
 609     },
 610     cpu => {
 611         optional => 1,
 612         description => "Emulated CPU type.",
 613         type => 'string',
 614         format => $cpu_fmt,
 615     },
 616     parent => get_standard_option('pve-snapshot-name', {
 617         optional => 1,
 618         description => "Parent snapshot name. This is used internally, and should not be modified.",
 619     }),
 620     snaptime => {
 621         optional => 1,
 622         description => "Timestamp for snapshots.",
 623         type => 'integer',
 624         minimum => 0,
 625     },
 626     vmstate => {
 627         optional => 1,
 628         type => 'string', format => 'pve-volume-id',
 629         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 630     },
 631     vmstatestorage => get_standard_option('pve-storage-id', {
 632         description => "Default storage for VM state volumes/files.",
 633         optional => 1,
 634     }),
 635     runningmachine => get_standard_option('pve-qemu-machine', {
 636         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 637     }),
 638     machine => get_standard_option('pve-qemu-machine'),
 639     arch => {
 640         description => "Virtual processor architecture. Defaults to the host.",
 641         optional => 1,
 642         type => 'string',
 643         enum => [qw(x86_64 aarch64)],
 644     },
 645     smbios1 => {
 646         description => "Specify SMBIOS type 1 fields.",
 647         type => 'string', format => 'pve-qm-smbios1',
 648         maxLength => 512,
 649         optional => 1,
 650     },
 651     protection => {
 652         optional => 1,
 653         type => 'boolean',
 654         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 655         default => 0,
 656     },
 657     bios => {
 658         optional => 1,
 659         type => 'string',
 660         enum => [ qw(seabios ovmf) ],
 661         description => "Select BIOS implementation.",
 662         default => 'seabios',
 663     },
 664     vmgenid => {
 665         type => 'string',
 666         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 667         format_description => 'UUID',
 668         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 669         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 670             " 128-bit integer value identifier to the guest OS. This allows to".
 671             " notify the guest operating system when the virtual machine is".
 672             " executed with a different configuration (e.g. snapshot execution".
 673             " or creation from a template). The guest operating system notices".
 674             " the change, and is then able to react as appropriate by marking".
 675             " its copies of distributed databases as dirty, re-initializing its".
 676             " random number generator, etc.\n".
 677             "Note that auto-creation only works when done throug API/CLI create".
 678             " or update methods, but not when manually editing the config file.",
 679         default => "1 (autogenerated)",
 680         optional => 1,
 681     },
 682     hookscript => {
 683         type => 'string',
 684         format => 'pve-volume-id',
 685         optional => 1,
 686         description => "Script that will be executed during various steps in the vms lifetime.",
 687     },
 688     ivshmem => {
 689         type => 'string',
 690         format => $ivshmem_fmt,
 691         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 692         optional => 1,
 693     },
 694     audio0 => {
 695         type => 'string',
 696         format => $audio_fmt,
 697         description => "Configure a audio device, useful in combination with QXL/Spice.",
 698         optional => 1
 699     },
 700     spice_enhancements => {
 701         type => 'string',
 702         format => $spice_enhancements_fmt,
 703         description => "Configure additional enhancements for SPICE.",
 704         optional => 1
 705     },
 706     tags => {
 707         type => 'string', format => 'pve-tag-list',
 708         description => 'Tags of the VM. This is only meta information.',
 709         optional => 1,
 710     },
 711 };
 712
 713 my $cicustom_fmt = {
 714     meta => {
 715         type => 'string',
 716         optional => 1,
 717         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 718         format => 'pve-volume-id',
 719         format_description => 'volume',
 720     },
 721     network => {
 722         type => 'string',
 723         optional => 1,
 724         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 725         format => 'pve-volume-id',
 726         format_description => 'volume',
 727     },
 728     user => {
 729         type => 'string',
 730         optional => 1,
 731         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 732         format => 'pve-volume-id',
 733         format_description => 'volume',
 734     },
 735 };
 736 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 737
 738 my $confdesc_cloudinit = {
 739     citype => {
 740         optional => 1,
 741         type => 'string',
 742         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 743         enum => ['configdrive2', 'nocloud'],
 744     },
 745     ciuser => {
 746         optional => 1,
 747         type => 'string',
 748         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 749     },
 750     cipassword => {
 751         optional => 1,
 752         type => 'string',
 753         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 754     },
 755     cicustom => {
 756         optional => 1,
 757         type => 'string',
 758         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 759         format => 'pve-qm-cicustom',
 760     },
 761     searchdomain => {
 762         optional => 1,
 763         type => 'string',
 764         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 765     },
 766     nameserver => {
 767         optional => 1,
 768         type => 'string', format => 'address-list',
 769         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 770     },
 771     sshkeys => {
 772         optional => 1,
 773         type => 'string',
 774         format => 'urlencoded',
 775         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 776     },
 777 };
 778
 779 # what about other qemu settings ?
 780 #cpu => 'string',
 781 #machine => 'string',
 782 #fda => 'file',
 783 #fdb => 'file',
 784 #mtdblock => 'file',
 785 #sd => 'file',
 786 #pflash => 'file',
 787 #snapshot => 'bool',
 788 #bootp => 'file',
 789 ##tftp => 'dir',
 790 ##smb => 'dir',
 791 #kernel => 'file',
 792 #append => 'string',
 793 #initrd => 'file',
 794 ##soundhw => 'string',
 795
 796 while (my ($k, $v) = each %$confdesc) {
 797     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 798 }
 799
 800 my $MAX_IDE_DISKS = 4;
 801 my $MAX_SCSI_DISKS = 14;
 802 my $MAX_VIRTIO_DISKS = 16;
 803 my $MAX_SATA_DISKS = 6;
 804 my $MAX_USB_DEVICES = 5;
 805 my $MAX_NETS = 32;
 806 my $MAX_UNUSED_DISKS = 256;
 807 my $MAX_HOSTPCI_DEVICES = 16;
 808 my $MAX_SERIAL_PORTS = 4;
 809 my $MAX_PARALLEL_PORTS = 3;
 810 my $MAX_NUMA = 8;
 811
 812 my $numa_fmt = {
 813     cpus => {
 814         type => "string",
 815         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 816         description => "CPUs accessing this NUMA node.",
 817         format_description => "id[-id];...",
 818     },
 819     memory => {
 820         type => "number",
 821         description => "Amount of memory this NUMA node provides.",
 822         optional => 1,
 823     },
 824     hostnodes => {
 825         type => "string",
 826         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 827         description => "Host NUMA nodes to use.",
 828         format_description => "id[-id];...",
 829         optional => 1,
 830     },
 831     policy => {
 832         type => 'string',
 833         enum => [qw(preferred bind interleave)],
 834         description => "NUMA allocation policy.",
 835         optional => 1,
 836     },
 837 };
 838 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 839 my $numadesc = {
 840     optional => 1,
 841     type => 'string', format => $numa_fmt,
 842     description => "NUMA topology.",
 843 };
 844 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 845
 846 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 847     $confdesc->{"numa$i"} = $numadesc;
 848 }
 849
 850 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 851                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 852                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 853 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 854
 855 my $net_fmt_bridge_descr = <<__EOD__;
 856 Bridge to attach the network device to. The Proxmox VE standard bridge
 857 is called 'vmbr0'.
 858
 859 If you do not specify a bridge, we create a kvm user (NATed) network
 860 device, which provides DHCP and DNS services. The following addresses
 861 are used:
 862
 863  10.0.2.2   Gateway
 864  10.0.2.3   DNS Server
 865  10.0.2.4   SMB Server
 866
 867 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 868 __EOD__
 869
 870 my $net_fmt = {
 871     macaddr  => get_standard_option('mac-addr', {
 872         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 873     }),
 874     model => {
 875         type => 'string',
 876         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 877         enum => $nic_model_list,
 878         default_key => 1,
 879     },
 880     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 881     bridge => {
 882         type => 'string',
 883         description => $net_fmt_bridge_descr,
 884         format_description => 'bridge',
 885         optional => 1,
 886     },
 887     queues => {
 888         type => 'integer',
 889         minimum => 0, maximum => 16,
 890         description => 'Number of packet queues to be used on the device.',
 891         optional => 1,
 892     },
 893     rate => {
 894         type => 'number',
 895         minimum => 0,
 896         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 897         optional => 1,
 898     },
 899     tag => {
 900         type => 'integer',
 901         minimum => 1, maximum => 4094,
 902         description => 'VLAN tag to apply to packets on this interface.',
 903         optional => 1,
 904     },
 905     trunks => {
 906         type => 'string',
 907         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 908         description => 'VLAN trunks to pass through this interface.',
 909         format_description => 'vlanid[;vlanid...]',
 910         optional => 1,
 911     },
 912     firewall => {
 913         type => 'boolean',
 914         description => 'Whether this interface should be protected by the firewall.',
 915         optional => 1,
 916     },
 917     link_down => {
 918         type => 'boolean',
 919         description => 'Whether this interface should be disconnected (like pulling the plug).',
 920         optional => 1,
 921     },
 922 };
 923
 924 my $netdesc = {
 925     optional => 1,
 926     type => 'string', format => $net_fmt,
 927     description => "Specify network devices.",
 928 };
 929
 930 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 931
 932 my $ipconfig_fmt = {
 933     ip => {
 934         type => 'string',
 935         format => 'pve-ipv4-config',
 936         format_description => 'IPv4Format/CIDR',
 937         description => 'IPv4 address in CIDR format.',
 938         optional => 1,
 939         default => 'dhcp',
 940     },
 941     gw => {
 942         type => 'string',
 943         format => 'ipv4',
 944         format_description => 'GatewayIPv4',
 945         description => 'Default gateway for IPv4 traffic.',
 946         optional => 1,
 947         requires => 'ip',
 948     },
 949     ip6 => {
 950         type => 'string',
 951         format => 'pve-ipv6-config',
 952         format_description => 'IPv6Format/CIDR',
 953         description => 'IPv6 address in CIDR format.',
 954         optional => 1,
 955         default => 'dhcp',
 956     },
 957     gw6 => {
 958         type => 'string',
 959         format => 'ipv6',
 960         format_description => 'GatewayIPv6',
 961         description => 'Default gateway for IPv6 traffic.',
 962         optional => 1,
 963         requires => 'ip6',
 964     },
 965 };
 966 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 967 my $ipconfigdesc = {
 968     optional => 1,
 969     type => 'string', format => 'pve-qm-ipconfig',
 970     description => <<'EODESCR',
 971 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 972
 973 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 974
 975 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 976 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 977
 978 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 979 EODESCR
 980 };
 981 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 982
 983 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 984     $confdesc->{"net$i"} = $netdesc;
 985     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 986 }
 987
 988 foreach my $key (keys %$confdesc_cloudinit) {
 989     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 990 }
 991
 992 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 993 sub verify_volume_id_or_qm_path {
 994     my ($volid, $noerr) = @_;
 995
 996     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 997         return $volid;
 998     }
 999
1000     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
1001     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1002     if ($@) {
1003         return undef if $noerr;
1004         die $@;
1005     }
1006     return $volid;
1007 }
1008
1009 my $drivename_hash;
1010
1011 my %drivedesc_base = (
1012     volume => { alias => 'file' },
1013     file => {
1014         type => 'string',
1015         format => 'pve-volume-id-or-qm-path',
1016         default_key => 1,
1017         format_description => 'volume',
1018         description => "The drive's backing volume.",
1019     },
1020     media => {
1021         type => 'string',
1022         enum => [qw(cdrom disk)],
1023         description => "The drive's media type.",
1024         default => 'disk',
1025         optional => 1
1026     },
1027     cyls => {
1028         type => 'integer',
1029         description => "Force the drive's physical geometry to have a specific cylinder count.",
1030         optional => 1
1031     },
1032     heads => {
1033         type => 'integer',
1034         description => "Force the drive's physical geometry to have a specific head count.",
1035         optional => 1
1036     },
1037     secs => {
1038         type => 'integer',
1039         description => "Force the drive's physical geometry to have a specific sector count.",
1040         optional => 1
1041     },
1042     trans => {
1043         type => 'string',
1044         enum => [qw(none lba auto)],
1045         description => "Force disk geometry bios translation mode.",
1046         optional => 1,
1047     },
1048     snapshot => {
1049         type => 'boolean',
1050         description => "Controls qemu's snapshot mode feature."
1051             . " If activated, changes made to the disk are temporary and will"
1052             . " be discarded when the VM is shutdown.",
1053         optional => 1,
1054     },
1055     cache => {
1056         type => 'string',
1057         enum => [qw(none writethrough writeback unsafe directsync)],
1058         description => "The drive's cache mode",
1059         optional => 1,
1060     },
1061     format => get_standard_option('pve-qm-image-format'),
1062     size => {
1063         type => 'string',
1064         format => 'disk-size',
1065         format_description => 'DiskSize',
1066         description => "Disk size. This is purely informational and has no effect.",
1067         optional => 1,
1068     },
1069     backup => {
1070         type => 'boolean',
1071         description => "Whether the drive should be included when making backups.",
1072         optional => 1,
1073     },
1074     replicate => {
1075         type => 'boolean',
1076         description => 'Whether the drive should considered for replication jobs.',
1077         optional => 1,
1078         default => 1,
1079     },
1080     rerror => {
1081         type => 'string',
1082         enum => [qw(ignore report stop)],
1083         description => 'Read error action.',
1084         optional => 1,
1085     },
1086     werror => {
1087         type => 'string',
1088         enum => [qw(enospc ignore report stop)],
1089         description => 'Write error action.',
1090         optional => 1,
1091     },
1092     aio => {
1093         type => 'string',
1094         enum => [qw(native threads)],
1095         description => 'AIO type to use.',
1096         optional => 1,
1097     },
1098     discard => {
1099         type => 'string',
1100         enum => [qw(ignore on)],
1101         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
1102         optional => 1,
1103     },
1104     detect_zeroes => {
1105         type => 'boolean',
1106         description => 'Controls whether to detect and try to optimize writes of zeroes.',
1107         optional => 1,
1108     },
1109     serial => {
1110         type => 'string',
1111         format => 'urlencoded',
1112         format_description => 'serial',
1113         maxLength => 20*3, # *3 since it's %xx url enoded
1114         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
1115         optional => 1,
1116     },
1117     shared => {
1118         type => 'boolean',
1119         description => 'Mark this locally-managed volume as available on all nodes',
1120         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1121         optional => 1,
1122         default => 0,
1123     }
1124 );
1125
1126 my %iothread_fmt = ( iothread => {
1127         type => 'boolean',
1128         description => "Whether to use iothreads for this drive",
1129         optional => 1,
1130 });
1131
1132 my %model_fmt = (
1133     model => {
1134         type => 'string',
1135         format => 'urlencoded',
1136         format_description => 'model',
1137         maxLength => 40*3, # *3 since it's %xx url enoded
1138         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1139         optional => 1,
1140     },
1141 );
1142
1143 my %queues_fmt = (
1144     queues => {
1145         type => 'integer',
1146         description => "Number of queues.",
1147         minimum => 2,
1148         optional => 1
1149     }
1150 );
1151
1152 my %scsiblock_fmt = (
1153     scsiblock => {
1154         type => 'boolean',
1155         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1156         optional => 1,
1157         default => 0,
1158     },
1159 );
1160
1161 my %ssd_fmt = (
1162     ssd => {
1163         type => 'boolean',
1164         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1165         optional => 1,
1166     },
1167 );
1168
1169 my %wwn_fmt = (
1170     wwn => {
1171         type => 'string',
1172         pattern => qr/^(0x)[0-9a-fA-F]{16}/,
1173         format_description => 'wwn',
1174         description => "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.",
1175         optional => 1,
1176     },
1177 );
1178
1179 my $add_throttle_desc = sub {
1180     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1181     my $d = {
1182         type => $type,
1183         format_description => $unit,
1184         description => "Maximum $what in $longunit.",
1185         optional => 1,
1186     };
1187     $d->{minimum} = $minimum if defined($minimum);
1188     $drivedesc_base{$key} = $d;
1189 };
1190 # throughput: (leaky bucket)
1191 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1192 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1193 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1194 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1195 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1196 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1197 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1198 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1199 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1200
1201 # pools: (pool of IO before throttling starts taking effect)
1202 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1203 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1204 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1205 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1206 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1207 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1208
1209 # burst lengths
1210 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1211 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1212 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1213 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1214 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1215 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1216
1217 # legacy support
1218 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1219 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1220 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1221 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1222
1223 my $ide_fmt = {
1224     %drivedesc_base,
1225     %model_fmt,
1226     %ssd_fmt,
1227     %wwn_fmt,
1228 };
1229 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1230
1231 my $idedesc = {
1232     optional => 1,
1233     type => 'string', format => $ide_fmt,
1234     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1235 };
1236 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1237
1238 my $scsi_fmt = {
1239     %drivedesc_base,
1240     %iothread_fmt,
1241     %queues_fmt,
1242     %scsiblock_fmt,
1243     %ssd_fmt,
1244     %wwn_fmt,
1245 };
1246 my $scsidesc = {
1247     optional => 1,
1248     type => 'string', format => $scsi_fmt,
1249     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1250 };
1251 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1252
1253 my $sata_fmt = {
1254     %drivedesc_base,
1255     %ssd_fmt,
1256     %wwn_fmt,
1257 };
1258 my $satadesc = {
1259     optional => 1,
1260     type => 'string', format => $sata_fmt,
1261     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1262 };
1263 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1264
1265 my $virtio_fmt = {
1266     %drivedesc_base,
1267     %iothread_fmt,
1268 };
1269 my $virtiodesc = {
1270     optional => 1,
1271     type => 'string', format => $virtio_fmt,
1272     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1273 };
1274 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1275
1276 my $alldrive_fmt = {
1277     %drivedesc_base,
1278     %iothread_fmt,
1279     %model_fmt,
1280     %queues_fmt,
1281     %scsiblock_fmt,
1282     %ssd_fmt,
1283     %wwn_fmt,
1284 };
1285
1286 my $efidisk_fmt = {
1287     volume => { alias => 'file' },
1288     file => {
1289         type => 'string',
1290         format => 'pve-volume-id-or-qm-path',
1291         default_key => 1,
1292         format_description => 'volume',
1293         description => "The drive's backing volume.",
1294     },
1295     format => get_standard_option('pve-qm-image-format'),
1296     size => {
1297         type => 'string',
1298         format => 'disk-size',
1299         format_description => 'DiskSize',
1300         description => "Disk size. This is purely informational and has no effect.",
1301         optional => 1,
1302     },
1303 };
1304
1305 my $efidisk_desc = {
1306     optional => 1,
1307     type => 'string', format => $efidisk_fmt,
1308     description => "Configure a Disk for storing EFI vars",
1309 };
1310
1311 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1312
1313 my $usb_fmt = {
1314     host => {
1315         default_key => 1,
1316         type => 'string', format => 'pve-qm-usb-device',
1317         format_description => 'HOSTUSBDEVICE|spice',
1318         description => <<EODESCR,
1319 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1320
1321  'bus-port(.port)*' (decimal numbers) or
1322  'vendor_id:product_id' (hexadeciaml numbers) or
1323  'spice'
1324
1325 You can use the 'lsusb -t' command to list existing usb devices.
1326
1327 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1328
1329 The value 'spice' can be used to add a usb redirection devices for spice.
1330 EODESCR
1331     },
1332     usb3 => {
1333         optional => 1,
1334         type => 'boolean',
1335         description => "Specifies whether if given host option is a USB3 device or port.",
1336         default => 0,
1337     },
1338 };
1339
1340 my $usbdesc = {
1341     optional => 1,
1342     type => 'string', format => $usb_fmt,
1343     description => "Configure an USB device (n is 0 to 4).",
1344 };
1345 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1346
1347 my $PCIRE = qr/([a-f0-9]{4}:)?[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1348 my $hostpci_fmt = {
1349     host => {
1350         default_key => 1,
1351         type => 'string',
1352         pattern => qr/$PCIRE(;$PCIRE)*/,
1353         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1354         description => <<EODESCR,
1355 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1356 of PCI virtual functions of the host. HOSTPCIID syntax is:
1357
1358 'bus:dev.func' (hexadecimal numbers)
1359
1360 You can us the 'lspci' command to list existing PCI devices.
1361 EODESCR
1362     },
1363     rombar => {
1364         type => 'boolean',
1365         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1366         optional => 1,
1367         default => 1,
1368     },
1369     romfile => {
1370         type => 'string',
1371         pattern => '[^,;]+',
1372         format_description => 'string',
1373         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1374         optional => 1,
1375     },
1376     pcie => {
1377         type => 'boolean',
1378         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1379         optional => 1,
1380         default => 0,
1381     },
1382     'x-vga' => {
1383         type => 'boolean',
1384         description =>  "Enable vfio-vga device support.",
1385         optional => 1,
1386         default => 0,
1387     },
1388     'mdev' => {
1389         type => 'string',
1390         format_description => 'string',
1391         pattern => '[^/\.:]+',
1392         optional => 1,
1393         description => <<EODESCR
1394 The type of mediated device to use.
1395 An instance of this type will be created on startup of the VM and
1396 will be cleaned up when the VM stops.
1397 EODESCR
1398     }
1399 };
1400 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1401
1402 my $hostpcidesc = {
1403         optional => 1,
1404         type => 'string', format => 'pve-qm-hostpci',
1405         description => "Map host PCI devices into guest.",
1406         verbose_description =>  <<EODESCR,
1407 Map host PCI devices into guest.
1408
1409 NOTE: This option allows direct access to host hardware. So it is no longer
1410 possible to migrate such machines - use with special care.
1411
1412 CAUTION: Experimental! User reported problems with this option.
1413 EODESCR
1414 };
1415 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1416
1417 my $serialdesc = {
1418         optional => 1,
1419         type => 'string',
1420         pattern => '(/dev/.+|socket)',
1421         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1422         verbose_description =>  <<EODESCR,
1423 Create a serial device inside the VM (n is 0 to 3), and pass through a
1424 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1425 host side (use 'qm terminal' to open a terminal connection).
1426
1427 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1428
1429 CAUTION: Experimental! User reported problems with this option.
1430 EODESCR
1431 };
1432
1433 my $paralleldesc= {
1434         optional => 1,
1435         type => 'string',
1436         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1437         description =>  "Map host parallel devices (n is 0 to 2).",
1438         verbose_description =>  <<EODESCR,
1439 Map host parallel devices (n is 0 to 2).
1440
1441 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1442
1443 CAUTION: Experimental! User reported problems with this option.
1444 EODESCR
1445 };
1446
1447 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1448     $confdesc->{"parallel$i"} = $paralleldesc;
1449 }
1450
1451 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1452     $confdesc->{"serial$i"} = $serialdesc;
1453 }
1454
1455 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1456     $confdesc->{"hostpci$i"} = $hostpcidesc;
1457 }
1458
1459 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1460     $drivename_hash->{"ide$i"} = 1;
1461     $confdesc->{"ide$i"} = $idedesc;
1462 }
1463
1464 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1465     $drivename_hash->{"sata$i"} = 1;
1466     $confdesc->{"sata$i"} = $satadesc;
1467 }
1468
1469 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1470     $drivename_hash->{"scsi$i"} = 1;
1471     $confdesc->{"scsi$i"} = $scsidesc ;
1472 }
1473
1474 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1475     $drivename_hash->{"virtio$i"} = 1;
1476     $confdesc->{"virtio$i"} = $virtiodesc;
1477 }
1478
1479 $drivename_hash->{efidisk0} = 1;
1480 $confdesc->{efidisk0} = $efidisk_desc;
1481
1482 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1483     $confdesc->{"usb$i"} = $usbdesc;
1484 }
1485
1486 my $unuseddesc = {
1487     optional => 1,
1488     type => 'string', format => 'pve-volume-id',
1489     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1490 };
1491
1492 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1493     $confdesc->{"unused$i"} = $unuseddesc;
1494 }
1495
1496 my $kvm_api_version = 0;
1497
1498 sub kvm_version {
1499     return $kvm_api_version if $kvm_api_version;
1500
1501     open my $fh, '<', '/dev/kvm'
1502         or return undef;
1503
1504     # 0xae00 => KVM_GET_API_VERSION
1505     $kvm_api_version = ioctl($fh, 0xae00, 0);
1506
1507     return $kvm_api_version;
1508 }
1509
1510 my $kvm_user_version = {};
1511 my $kvm_mtime = {};
1512
1513 sub kvm_user_version {
1514     my ($binary) = @_;
1515
1516     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1517     my $st = stat($binary);
1518
1519     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1520     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1521         $cachedmtime == $st->mtime;
1522
1523     $kvm_user_version->{$binary} = 'unknown';
1524     $kvm_mtime->{$binary} = $st->mtime;
1525
1526     my $code = sub {
1527         my $line = shift;
1528         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1529             $kvm_user_version->{$binary} = $2;
1530         }
1531     };
1532
1533     eval { run_command([$binary, '--version'], outfunc => $code); };
1534     warn $@ if $@;
1535
1536     return $kvm_user_version->{$binary};
1537
1538 }
1539
1540 sub kernel_has_vhost_net {
1541     return -c '/dev/vhost-net';
1542 }
1543
1544 sub valid_drive_names {
1545     # order is important - used to autoselect boot disk
1546     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1547             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1548             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1549             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1550             'efidisk0');
1551 }
1552
1553 sub is_valid_drivename {
1554     my $dev = shift;
1555
1556     return defined($drivename_hash->{$dev});
1557 }
1558
1559 sub option_exists {
1560     my $key = shift;
1561     return defined($confdesc->{$key});
1562 }
1563
1564 my $cdrom_path;
1565 sub get_cdrom_path {
1566
1567     return  $cdrom_path if $cdrom_path;
1568
1569     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1570     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1571     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1572 }
1573
1574 sub get_iso_path {
1575     my ($storecfg, $vmid, $cdrom) = @_;
1576
1577     if ($cdrom eq 'cdrom') {
1578         return get_cdrom_path();
1579     } elsif ($cdrom eq 'none') {
1580         return '';
1581     } elsif ($cdrom =~ m|^/|) {
1582         return $cdrom;
1583     } else {
1584         return PVE::Storage::path($storecfg, $cdrom);
1585     }
1586 }
1587
1588 # try to convert old style file names to volume IDs
1589 sub filename_to_volume_id {
1590     my ($vmid, $file, $media) = @_;
1591
1592      if (!($file eq 'none' || $file eq 'cdrom' ||
1593           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1594
1595         return undef if $file =~ m|/|;
1596
1597         if ($media && $media eq 'cdrom') {
1598             $file = "local:iso/$file";
1599         } else {
1600             $file = "local:$vmid/$file";
1601         }
1602     }
1603
1604     return $file;
1605 }
1606
1607 sub verify_media_type {
1608     my ($opt, $vtype, $media) = @_;
1609
1610     return if !$media;
1611
1612     my $etype;
1613     if ($media eq 'disk') {
1614         $etype = 'images';
1615     } elsif ($media eq 'cdrom') {
1616         $etype = 'iso';
1617     } else {
1618         die "internal error";
1619     }
1620
1621     return if ($vtype eq $etype);
1622
1623     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1624 }
1625
1626 sub cleanup_drive_path {
1627     my ($opt, $storecfg, $drive) = @_;
1628
1629     # try to convert filesystem paths to volume IDs
1630
1631     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1632         ($drive->{file} !~ m|^/dev/.+|) &&
1633         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1634         ($drive->{file} !~ m/^\d+$/)) {
1635         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1636         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1637         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1638         verify_media_type($opt, $vtype, $drive->{media});
1639         $drive->{file} = $volid;
1640     }
1641
1642     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1643 }
1644
1645 sub parse_hotplug_features {
1646     my ($data) = @_;
1647
1648     my $res = {};
1649
1650     return $res if $data eq '0';
1651
1652     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1653
1654     foreach my $feature (PVE::Tools::split_list($data)) {
1655         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1656             $res->{$1} = 1;
1657         } else {
1658             die "invalid hotplug feature '$feature'\n";
1659         }
1660     }
1661     return $res;
1662 }
1663
1664 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1665 sub pve_verify_hotplug_features {
1666     my ($value, $noerr) = @_;
1667
1668     return $value if parse_hotplug_features($value);
1669
1670     return undef if $noerr;
1671
1672     die "unable to parse hotplug option\n";
1673 }
1674
1675 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1676 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1677 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1678 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1679 #        [,iothread=on][,serial=serial][,model=model]
1680
1681 sub parse_drive {
1682     my ($key, $data) = @_;
1683
1684     my ($interface, $index);
1685
1686     if ($key =~ m/^([^\d]+)(\d+)$/) {
1687         $interface = $1;
1688         $index = $2;
1689     } else {
1690         return undef;
1691     }
1692
1693     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1694                                      : $confdesc->{$key}->{format};
1695     if (!$desc) {
1696         warn "invalid drive key: $key\n";
1697         return undef;
1698     }
1699     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1700     return undef if !$res;
1701     $res->{interface} = $interface;
1702     $res->{index} = $index;
1703
1704     my $error = 0;
1705     foreach my $opt (qw(bps bps_rd bps_wr)) {
1706         if (my $bps = defined(delete $res->{$opt})) {
1707             if (defined($res->{"m$opt"})) {
1708                 warn "both $opt and m$opt specified\n";
1709                 ++$error;
1710                 next;
1711             }
1712             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1713         }
1714     }
1715
1716     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1717     for my $requirement (
1718         [mbps_max => 'mbps'],
1719         [mbps_rd_max => 'mbps_rd'],
1720         [mbps_wr_max => 'mbps_wr'],
1721         [miops_max => 'miops'],
1722         [miops_rd_max => 'miops_rd'],
1723         [miops_wr_max => 'miops_wr'],
1724         [bps_max_length => 'mbps_max'],
1725         [bps_rd_max_length => 'mbps_rd_max'],
1726         [bps_wr_max_length => 'mbps_wr_max'],
1727         [iops_max_length => 'iops_max'],
1728         [iops_rd_max_length => 'iops_rd_max'],
1729         [iops_wr_max_length => 'iops_wr_max']) {
1730         my ($option, $requires) = @$requirement;
1731         if ($res->{$option} && !$res->{$requires}) {
1732             warn "$option requires $requires\n";
1733             ++$error;
1734         }
1735     }
1736
1737     return undef if $error;
1738
1739     return undef if $res->{mbps_rd} && $res->{mbps};
1740     return undef if $res->{mbps_wr} && $res->{mbps};
1741     return undef if $res->{iops_rd} && $res->{iops};
1742     return undef if $res->{iops_wr} && $res->{iops};
1743
1744     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1745         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1746         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1747         return undef if $res->{interface} eq 'virtio';
1748     }
1749
1750     if (my $size = $res->{size}) {
1751         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1752     }
1753
1754     return $res;
1755 }
1756
1757 sub print_drive {
1758     my ($drive) = @_;
1759     my $data = { %$drive };
1760     delete $data->{$_} for qw(index interface);
1761     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1762 }
1763
1764 sub scsi_inquiry {
1765     my($fh, $noerr) = @_;
1766
1767     my $SG_IO = 0x2285;
1768     my $SG_GET_VERSION_NUM = 0x2282;
1769
1770     my $versionbuf = "\x00" x 8;
1771     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1772     if (!$ret) {
1773         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1774         return undef;
1775     }
1776     my $version = unpack("I", $versionbuf);
1777     if ($version < 30000) {
1778         die "scsi generic interface too old\n"  if !$noerr;
1779         return undef;
1780     }
1781
1782     my $buf = "\x00" x 36;
1783     my $sensebuf = "\x00" x 8;
1784     my $cmd = pack("C x3 C x1", 0x12, 36);
1785
1786     # see /usr/include/scsi/sg.h
1787     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1788
1789     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1790                       length($sensebuf), 0, length($buf), $buf,
1791                       $cmd, $sensebuf, 6000);
1792
1793     $ret = ioctl($fh, $SG_IO, $packet);
1794     if (!$ret) {
1795         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1796         return undef;
1797     }
1798
1799     my @res = unpack($sg_io_hdr_t, $packet);
1800     if ($res[17] || $res[18]) {
1801         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1802         return undef;
1803     }
1804
1805     my $res = {};
1806     (my $byte0, my $byte1, $res->{vendor},
1807      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1808
1809     $res->{removable} = $byte1 & 128 ? 1 : 0;
1810     $res->{type} = $byte0 & 31;
1811
1812     return $res;
1813 }
1814
1815 sub path_is_scsi {
1816     my ($path) = @_;
1817
1818     my $fh = IO::File->new("+<$path") || return undef;
1819     my $res = scsi_inquiry($fh, 1);
1820     close($fh);
1821
1822     return $res;
1823 }
1824
1825 sub print_tabletdevice_full {
1826     my ($conf, $arch) = @_;
1827
1828     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1829
1830     # we use uhci for old VMs because tablet driver was buggy in older qemu
1831     my $usbbus;
1832     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1833         $usbbus = 'ehci';
1834     } else {
1835         $usbbus = 'uhci';
1836     }
1837
1838     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1839 }
1840
1841 sub print_keyboarddevice_full {
1842     my ($conf, $arch, $machine) = @_;
1843
1844     return undef if $arch ne 'aarch64';
1845
1846     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1847 }
1848
1849 sub print_drivedevice_full {
1850     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1851
1852     my $device = '';
1853     my $maxdev = 0;
1854
1855     if ($drive->{interface} eq 'virtio') {
1856         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1857         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1858         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1859     } elsif ($drive->{interface} eq 'scsi') {
1860
1861         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1862         my $unit = $drive->{index} % $maxdev;
1863         my $devicetype = 'hd';
1864         my $path = '';
1865         if (drive_is_cdrom($drive)) {
1866             $devicetype = 'cd';
1867         } else {
1868             if ($drive->{file} =~ m|^/|) {
1869                 $path = $drive->{file};
1870                 if (my $info = path_is_scsi($path)) {
1871                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1872                         $devicetype = 'block';
1873                     } elsif ($info->{type} == 1) { # tape
1874                         $devicetype = 'generic';
1875                     }
1876                 }
1877             } else {
1878                  $path = PVE::Storage::path($storecfg, $drive->{file});
1879             }
1880
1881             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1882             my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
1883             if ($path =~ m/^iscsi\:\/\// &&
1884                !min_version($version, 4, 1)) {
1885                 $devicetype = 'generic';
1886             }
1887         }
1888
1889         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1890             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1891         } else {
1892             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1893         }
1894
1895         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1896             $device .= ",rotation_rate=1";
1897         }
1898         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1899
1900     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1901         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1902         my $controller = int($drive->{index} / $maxdev);
1903         my $unit = $drive->{index} % $maxdev;
1904         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1905
1906         $device = "ide-$devicetype";
1907         if ($drive->{interface} eq 'ide') {
1908             $device .= ",bus=ide.$controller,unit=$unit";
1909         } else {
1910             $device .= ",bus=ahci$controller.$unit";
1911         }
1912         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1913
1914         if ($devicetype eq 'hd') {
1915             if (my $model = $drive->{model}) {
1916                 $model = URI::Escape::uri_unescape($model);
1917                 $device .= ",model=$model";
1918             }
1919             if ($drive->{ssd}) {
1920                 $device .= ",rotation_rate=1";
1921             }
1922         }
1923         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1924     } elsif ($drive->{interface} eq 'usb') {
1925         die "implement me";
1926         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1927     } else {
1928         die "unsupported interface type";
1929     }
1930
1931     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1932
1933     if (my $serial = $drive->{serial}) {
1934         $serial = URI::Escape::uri_unescape($serial);
1935         $device .= ",serial=$serial";
1936     }
1937
1938
1939     return $device;
1940 }
1941
1942 sub get_initiator_name {
1943     my $initiator;
1944
1945     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1946     while (defined(my $line = <$fh>)) {
1947         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1948         $initiator = $1;
1949         last;
1950     }
1951     $fh->close();
1952
1953     return $initiator;
1954 }
1955
1956 sub print_drive_full {
1957     my ($storecfg, $vmid, $drive) = @_;
1958
1959     my $path;
1960     my $volid = $drive->{file};
1961     my $format;
1962
1963     if (drive_is_cdrom($drive)) {
1964         $path = get_iso_path($storecfg, $vmid, $volid);
1965     } else {
1966         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1967         if ($storeid) {
1968             $path = PVE::Storage::path($storecfg, $volid);
1969             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1970             $format = qemu_img_format($scfg, $volname);
1971         } else {
1972             $path = $volid;
1973             $format = "raw";
1974         }
1975    }
1976
1977     my $opts = '';
1978     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1979     foreach my $o (@qemu_drive_options) {
1980         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1981     }
1982
1983     # snapshot only accepts on|off
1984     if (defined($drive->{snapshot})) {
1985         my $v = $drive->{snapshot} ? 'on' : 'off';
1986         $opts .= ",snapshot=$v";
1987     }
1988
1989     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1990         my ($dir, $qmpname) = @$type;
1991         if (my $v = $drive->{"mbps$dir"}) {
1992             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1993         }
1994         if (my $v = $drive->{"mbps${dir}_max"}) {
1995             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1996         }
1997         if (my $v = $drive->{"bps${dir}_max_length"}) {
1998             $opts .= ",throttling.bps$qmpname-max-length=$v";
1999         }
2000         if (my $v = $drive->{"iops${dir}"}) {
2001             $opts .= ",throttling.iops$qmpname=$v";
2002         }
2003         if (my $v = $drive->{"iops${dir}_max"}) {
2004             $opts .= ",throttling.iops$qmpname-max=$v";
2005         }
2006         if (my $v = $drive->{"iops${dir}_max_length"}) {
2007             $opts .= ",throttling.iops$qmpname-max-length=$v";
2008         }
2009     }
2010
2011     $opts .= ",format=$format" if $format && !$drive->{format};
2012
2013     my $cache_direct = 0;
2014
2015     if (my $cache = $drive->{cache}) {
2016         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
2017     } elsif (!drive_is_cdrom($drive)) {
2018         $opts .= ",cache=none";
2019         $cache_direct = 1;
2020     }
2021
2022     # aio native works only with O_DIRECT
2023     if (!$drive->{aio}) {
2024         if($cache_direct) {
2025             $opts .= ",aio=native";
2026         } else {
2027             $opts .= ",aio=threads";
2028         }
2029     }
2030
2031     if (!drive_is_cdrom($drive)) {
2032         my $detectzeroes;
2033         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
2034             $detectzeroes = 'off';
2035         } elsif ($drive->{discard}) {
2036             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
2037         } else {
2038             # This used to be our default with discard not being specified:
2039             $detectzeroes = 'on';
2040         }
2041         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
2042     }
2043
2044     my $pathinfo = $path ? "file=$path," : '';
2045
2046     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
2047 }
2048
2049 sub print_netdevice_full {
2050     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
2051
2052     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2053
2054     my $device = $net->{model};
2055     if ($net->{model} eq 'virtio') {
2056          $device = 'virtio-net-pci';
2057      };
2058
2059     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
2060     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
2061     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
2062         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
2063         my $vectors = $net->{queues} * 2 + 2;
2064         $tmpstr .= ",vectors=$vectors,mq=on";
2065     }
2066     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
2067
2068     if ($use_old_bios_files) {
2069         my $romfile;
2070         if ($device eq 'virtio-net-pci') {
2071             $romfile = 'pxe-virtio.rom';
2072         } elsif ($device eq 'e1000') {
2073             $romfile = 'pxe-e1000.rom';
2074         } elsif ($device eq 'ne2k') {
2075             $romfile = 'pxe-ne2k_pci.rom';
2076         } elsif ($device eq 'pcnet') {
2077             $romfile = 'pxe-pcnet.rom';
2078         } elsif ($device eq 'rtl8139') {
2079             $romfile = 'pxe-rtl8139.rom';
2080         }
2081         $tmpstr .= ",romfile=$romfile" if $romfile;
2082     }
2083
2084     return $tmpstr;
2085 }
2086
2087 sub print_netdev_full {
2088     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
2089
2090     my $i = '';
2091     if ($netid =~ m/^net(\d+)$/) {
2092         $i = int($1);
2093     }
2094
2095     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
2096
2097     my $ifname = "tap${vmid}i$i";
2098
2099     # kvm uses TUNSETIFF ioctl, and that limits ifname length
2100     die "interface name '$ifname' is too long (max 15 character)\n"
2101         if length($ifname) >= 16;
2102
2103     my $vhostparam = '';
2104     if (is_native($arch)) {
2105         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
2106     }
2107
2108     my $vmname = $conf->{name} || "vm$vmid";
2109
2110     my $netdev = "";
2111     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
2112
2113     if ($net->{bridge}) {
2114         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
2115     } else {
2116         $netdev = "type=user,id=$netid,hostname=$vmname";
2117     }
2118
2119     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
2120
2121     return $netdev;
2122 }
2123
2124
2125 sub print_cpu_device {
2126     my ($conf, $id) = @_;
2127
2128     my $kvm = $conf->{kvm} // 1;
2129     my $cpu = $kvm ? "kvm64" : "qemu64";
2130     if (my $cputype = $conf->{cpu}) {
2131         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
2132             or die "Cannot parse cpu description: $cputype\n";
2133         $cpu = $cpuconf->{cputype};
2134     }
2135
2136     my $cores = $conf->{cores} || 1;
2137
2138     my $current_core = ($id - 1) % $cores;
2139     my $current_socket = int(($id - 1 - $current_core)/$cores);
2140
2141     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2142 }
2143
2144 my $vga_map = {
2145     'cirrus' => 'cirrus-vga',
2146     'std' => 'VGA',
2147     'vmware' => 'vmware-svga',
2148     'virtio' => 'virtio-vga',
2149 };
2150
2151 sub print_vga_device {
2152     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
2153
2154     my $type = $vga_map->{$vga->{type}};
2155     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2156         $type = 'virtio-gpu';
2157     }
2158     my $vgamem_mb = $vga->{memory};
2159
2160     my $max_outputs = '';
2161     if ($qxlnum) {
2162         $type = $id ? 'qxl' : 'qxl-vga';
2163
2164         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
2165             # set max outputs so linux can have up to 4 qxl displays with one device
2166             if (min_version($machine_version, 4, 1)) {
2167                 $max_outputs = ",max_outputs=4";
2168             }
2169         }
2170     }
2171
2172     die "no devicetype for $vga->{type}\n" if !$type;
2173
2174     my $memory = "";
2175     if ($vgamem_mb) {
2176         if ($vga->{type} eq 'virtio') {
2177             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2178             $memory = ",max_hostmem=$bytes";
2179         } elsif ($qxlnum) {
2180             # from https://www.spice-space.org/multiple-monitors.html
2181             $memory = ",vgamem_mb=$vga->{memory}";
2182             my $ram = $vgamem_mb * 4;
2183             my $vram = $vgamem_mb * 2;
2184             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2185         } else {
2186             $memory = ",vgamem_mb=$vga->{memory}";
2187         }
2188     } elsif ($qxlnum && $id) {
2189         $memory = ",ram_size=67108864,vram_size=33554432";
2190     }
2191
2192     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2193     my $vgaid = "vga" . ($id // '');
2194     my $pciaddr;
2195
2196     if ($q35 && $vgaid eq 'vga') {
2197         # the first display uses pcie.0 bus on q35 machines
2198         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2199     } else {
2200         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2201     }
2202
2203     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}";
2204 }
2205
2206 sub drive_is_cloudinit {
2207     my ($drive) = @_;
2208     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2209 }
2210
2211 sub drive_is_cdrom {
2212     my ($drive, $exclude_cloudinit) = @_;
2213
2214     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2215
2216     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2217
2218 }
2219
2220 sub parse_number_sets {
2221     my ($set) = @_;
2222     my $res = [];
2223     foreach my $part (split(/;/, $set)) {
2224         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2225             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2226             push @$res, [ $1, $2 ];
2227         } else {
2228             die "invalid range: $part\n";
2229         }
2230     }
2231     return $res;
2232 }
2233
2234 sub parse_numa {
2235     my ($data) = @_;
2236
2237     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2238     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2239     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2240     return $res;
2241 }
2242
2243 sub parse_hostpci {
2244     my ($value) = @_;
2245
2246     return undef if !$value;
2247
2248     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2249
2250     my @idlist = split(/;/, $res->{host});
2251     delete $res->{host};
2252     foreach my $id (@idlist) {
2253         my $devs = PVE::SysFSTools::lspci($id);
2254         die "no PCI device found for '$id'\n" if !scalar(@$devs);
2255         push @{$res->{pciid}}, @$devs;
2256     }
2257     return $res;
2258 }
2259
2260 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2261 sub parse_net {
2262     my ($data) = @_;
2263
2264     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2265     if ($@) {
2266         warn $@;
2267         return undef;
2268     }
2269     if (!defined($res->{macaddr})) {
2270         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2271         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2272     }
2273     return $res;
2274 }
2275
2276 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2277 sub parse_ipconfig {
2278     my ($data) = @_;
2279
2280     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2281     if ($@) {
2282         warn $@;
2283         return undef;
2284     }
2285
2286     if ($res->{gw} && !$res->{ip}) {
2287         warn 'gateway specified without specifying an IP address';
2288         return undef;
2289     }
2290     if ($res->{gw6} && !$res->{ip6}) {
2291         warn 'IPv6 gateway specified without specifying an IPv6 address';
2292         return undef;
2293     }
2294     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2295         warn 'gateway specified together with DHCP';
2296         return undef;
2297     }
2298     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2299         # gw6 + auto/dhcp
2300         warn "IPv6 gateway specified together with $res->{ip6} address";
2301         return undef;
2302     }
2303
2304     if (!$res->{ip} && !$res->{ip6}) {
2305         return { ip => 'dhcp', ip6 => 'dhcp' };
2306     }
2307
2308     return $res;
2309 }
2310
2311 sub print_net {
2312     my $net = shift;
2313
2314     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2315 }
2316
2317 sub add_random_macs {
2318     my ($settings) = @_;
2319
2320     foreach my $opt (keys %$settings) {
2321         next if $opt !~ m/^net(\d+)$/;
2322         my $net = parse_net($settings->{$opt});
2323         next if !$net;
2324         $settings->{$opt} = print_net($net);
2325     }
2326 }
2327
2328 sub vm_is_volid_owner {
2329     my ($storecfg, $vmid, $volid) = @_;
2330
2331     if ($volid !~  m|^/|) {
2332         my ($path, $owner);
2333         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2334         if ($owner && ($owner == $vmid)) {
2335             return 1;
2336         }
2337     }
2338
2339     return undef;
2340 }
2341
2342 sub vmconfig_register_unused_drive {
2343     my ($storecfg, $vmid, $conf, $drive) = @_;
2344
2345     if (drive_is_cloudinit($drive)) {
2346         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2347         warn $@ if $@;
2348     } elsif (!drive_is_cdrom($drive)) {
2349         my $volid = $drive->{file};
2350         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2351             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2352         }
2353     }
2354 }
2355
2356 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2357 my $smbios1_fmt = {
2358     uuid => {
2359         type => 'string',
2360         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2361         format_description => 'UUID',
2362         description => "Set SMBIOS1 UUID.",
2363         optional => 1,
2364     },
2365     version => {
2366         type => 'string',
2367         pattern => '[A-Za-z0-9+\/]+={0,2}',
2368         format_description => 'Base64 encoded string',
2369         description => "Set SMBIOS1 version.",
2370         optional => 1,
2371     },
2372     serial => {
2373         type => 'string',
2374         pattern => '[A-Za-z0-9+\/]+={0,2}',
2375         format_description => 'Base64 encoded string',
2376         description => "Set SMBIOS1 serial number.",
2377         optional => 1,
2378     },
2379     manufacturer => {
2380         type => 'string',
2381         pattern => '[A-Za-z0-9+\/]+={0,2}',
2382         format_description => 'Base64 encoded string',
2383         description => "Set SMBIOS1 manufacturer.",
2384         optional => 1,
2385     },
2386     product => {
2387         type => 'string',
2388         pattern => '[A-Za-z0-9+\/]+={0,2}',
2389         format_description => 'Base64 encoded string',
2390         description => "Set SMBIOS1 product ID.",
2391         optional => 1,
2392     },
2393     sku => {
2394         type => 'string',
2395         pattern => '[A-Za-z0-9+\/]+={0,2}',
2396         format_description => 'Base64 encoded string',
2397         description => "Set SMBIOS1 SKU string.",
2398         optional => 1,
2399     },
2400     family => {
2401         type => 'string',
2402         pattern => '[A-Za-z0-9+\/]+={0,2}',
2403         format_description => 'Base64 encoded string',
2404         description => "Set SMBIOS1 family string.",
2405         optional => 1,
2406     },
2407     base64 => {
2408         type => 'boolean',
2409         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2410         optional => 1,
2411     },
2412 };
2413
2414 sub parse_smbios1 {
2415     my ($data) = @_;
2416
2417     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2418     warn $@ if $@;
2419     return $res;
2420 }
2421
2422 sub print_smbios1 {
2423     my ($smbios1) = @_;
2424     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2425 }
2426
2427 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2428
2429 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2430 sub verify_bootdisk {
2431     my ($value, $noerr) = @_;
2432
2433     return $value if is_valid_drivename($value);
2434
2435     return undef if $noerr;
2436
2437     die "invalid boot disk '$value'\n";
2438 }
2439
2440 sub parse_watchdog {
2441     my ($value) = @_;
2442
2443     return undef if !$value;
2444
2445     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2446     warn $@ if $@;
2447     return $res;
2448 }
2449
2450 sub parse_guest_agent {
2451     my ($value) = @_;
2452
2453     return {} if !defined($value->{agent});
2454
2455     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2456     warn $@ if $@;
2457
2458     # if the agent is disabled ignore the other potentially set properties
2459     return {} if !$res->{enabled};
2460     return $res;
2461 }
2462
2463 sub parse_vga {
2464     my ($value) = @_;
2465
2466     return {} if !$value;
2467     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2468     warn $@ if $@;
2469     return $res;
2470 }
2471
2472 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2473 sub verify_usb_device {
2474     my ($value, $noerr) = @_;
2475
2476     return $value if parse_usb_device($value);
2477
2478     return undef if $noerr;
2479
2480     die "unable to parse usb device\n";
2481 }
2482
2483 # add JSON properties for create and set function
2484 sub json_config_properties {
2485     my $prop = shift;
2486
2487     foreach my $opt (keys %$confdesc) {
2488         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2489         $prop->{$opt} = $confdesc->{$opt};
2490     }
2491
2492     return $prop;
2493 }
2494
2495 # return copy of $confdesc_cloudinit to generate documentation
2496 sub cloudinit_config_properties {
2497
2498     return dclone($confdesc_cloudinit);
2499 }
2500
2501 sub check_type {
2502     my ($key, $value) = @_;
2503
2504     die "unknown setting '$key'\n" if !$confdesc->{$key};
2505
2506     my $type = $confdesc->{$key}->{type};
2507
2508     if (!defined($value)) {
2509         die "got undefined value\n";
2510     }
2511
2512     if ($value =~ m/[\n\r]/) {
2513         die "property contains a line feed\n";
2514     }
2515
2516     if ($type eq 'boolean') {
2517         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2518         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2519         die "type check ('boolean') failed - got '$value'\n";
2520     } elsif ($type eq 'integer') {
2521         return int($1) if $value =~ m/^(\d+)$/;
2522         die "type check ('integer') failed - got '$value'\n";
2523     } elsif ($type eq 'number') {
2524         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2525         die "type check ('number') failed - got '$value'\n";
2526     } elsif ($type eq 'string') {
2527         if (my $fmt = $confdesc->{$key}->{format}) {
2528             PVE::JSONSchema::check_format($fmt, $value);
2529             return $value;
2530         }
2531         $value =~ s/^\"(.*)\"$/$1/;
2532         return $value;
2533     } else {
2534         die "internal error"
2535     }
2536 }
2537
2538 sub destroy_vm {
2539     my ($storecfg, $vmid, $skiplock, $replacement_conf) = @_;
2540
2541     my $conf = PVE::QemuConfig->load_config($vmid);
2542
2543     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2544
2545     if ($conf->{template}) {
2546         # check if any base image is still used by a linked clone
2547         foreach_drive($conf, sub {
2548                 my ($ds, $drive) = @_;
2549                 return if drive_is_cdrom($drive);
2550
2551                 my $volid = $drive->{file};
2552                 return if !$volid || $volid =~ m|^/|;
2553
2554                 die "base volume '$volid' is still in use by linked cloned\n"
2555                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2556
2557         });
2558     }
2559
2560     # only remove disks owned by this VM
2561     foreach_drive($conf, sub {
2562         my ($ds, $drive) = @_;
2563         return if drive_is_cdrom($drive, 1);
2564
2565         my $volid = $drive->{file};
2566         return if !$volid || $volid =~ m|^/|;
2567
2568         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2569         return if !$path || !$owner || ($owner != $vmid);
2570
2571         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2572         warn "Could not remove disk '$volid', check manually: $@" if $@;
2573     });
2574
2575     # also remove unused disk
2576     my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2577     PVE::Storage::foreach_volid($vmdisks, sub {
2578         my ($volid, $sid, $volname, $d) = @_;
2579         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2580         warn $@ if $@;
2581     });
2582
2583     if (defined $replacement_conf) {
2584         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2585     } else {
2586         PVE::QemuConfig->destroy_config($vmid);
2587     }
2588 }
2589
2590 sub parse_vm_config {
2591     my ($filename, $raw) = @_;
2592
2593     return undef if !defined($raw);
2594
2595     my $res = {
2596         digest => Digest::SHA::sha1_hex($raw),
2597         snapshots => {},
2598         pending => {},
2599     };
2600
2601     $filename =~ m|/qemu-server/(\d+)\.conf$|
2602         || die "got strange filename '$filename'";
2603
2604     my $vmid = $1;
2605
2606     my $conf = $res;
2607     my $descr;
2608     my $section = '';
2609
2610     my @lines = split(/\n/, $raw);
2611     foreach my $line (@lines) {
2612         next if $line =~ m/^\s*$/;
2613
2614         if ($line =~ m/^\[PENDING\]\s*$/i) {
2615             $section = 'pending';
2616             if (defined($descr)) {
2617                 $descr =~ s/\s+$//;
2618                 $conf->{description} = $descr;
2619             }
2620             $descr = undef;
2621             $conf = $res->{$section} = {};
2622             next;
2623
2624         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2625             $section = $1;
2626             if (defined($descr)) {
2627                 $descr =~ s/\s+$//;
2628                 $conf->{description} = $descr;
2629             }
2630             $descr = undef;
2631             $conf = $res->{snapshots}->{$section} = {};
2632             next;
2633         }
2634
2635         if ($line =~ m/^\#(.*)\s*$/) {
2636             $descr = '' if !defined($descr);
2637             $descr .= PVE::Tools::decode_text($1) . "\n";
2638             next;
2639         }
2640
2641         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2642             $descr = '' if !defined($descr);
2643             $descr .= PVE::Tools::decode_text($2);
2644         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2645             $conf->{snapstate} = $1;
2646         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2647             my $key = $1;
2648             my $value = $2;
2649             $conf->{$key} = $value;
2650         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2651             my $value = $1;
2652             if ($section eq 'pending') {
2653                 $conf->{delete} = $value; # we parse this later
2654             } else {
2655                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2656             }
2657         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2658             my $key = $1;
2659             my $value = $2;
2660             eval { $value = check_type($key, $value); };
2661             if ($@) {
2662                 warn "vm $vmid - unable to parse value of '$key' - $@";
2663             } else {
2664                 $key = 'ide2' if $key eq 'cdrom';
2665                 my $fmt = $confdesc->{$key}->{format};
2666                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2667                     my $v = parse_drive($key, $value);
2668                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2669                         $v->{file} = $volid;
2670                         $value = print_drive($v);
2671                     } else {
2672                         warn "vm $vmid - unable to parse value of '$key'\n";
2673                         next;
2674                     }
2675                 }
2676
2677                 $conf->{$key} = $value;
2678             }
2679         }
2680     }
2681
2682     if (defined($descr)) {
2683         $descr =~ s/\s+$//;
2684         $conf->{description} = $descr;
2685     }
2686     delete $res->{snapstate}; # just to be sure
2687
2688     return $res;
2689 }
2690
2691 sub write_vm_config {
2692     my ($filename, $conf) = @_;
2693
2694     delete $conf->{snapstate}; # just to be sure
2695
2696     if ($conf->{cdrom}) {
2697         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2698         $conf->{ide2} = $conf->{cdrom};
2699         delete $conf->{cdrom};
2700     }
2701
2702     # we do not use 'smp' any longer
2703     if ($conf->{sockets}) {
2704         delete $conf->{smp};
2705     } elsif ($conf->{smp}) {
2706         $conf->{sockets} = $conf->{smp};
2707         delete $conf->{cores};
2708         delete $conf->{smp};
2709     }
2710
2711     my $used_volids = {};
2712
2713     my $cleanup_config = sub {
2714         my ($cref, $pending, $snapname) = @_;
2715
2716         foreach my $key (keys %$cref) {
2717             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2718                 $key eq 'snapstate' || $key eq 'pending';
2719             my $value = $cref->{$key};
2720             if ($key eq 'delete') {
2721                 die "propertry 'delete' is only allowed in [PENDING]\n"
2722                     if !$pending;
2723                 # fixme: check syntax?
2724                 next;
2725             }
2726             eval { $value = check_type($key, $value); };
2727             die "unable to parse value of '$key' - $@" if $@;
2728
2729             $cref->{$key} = $value;
2730
2731             if (!$snapname && is_valid_drivename($key)) {
2732                 my $drive = parse_drive($key, $value);
2733                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2734             }
2735         }
2736     };
2737
2738     &$cleanup_config($conf);
2739
2740     &$cleanup_config($conf->{pending}, 1);
2741
2742     foreach my $snapname (keys %{$conf->{snapshots}}) {
2743         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2744         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2745     }
2746
2747     # remove 'unusedX' settings if we re-add a volume
2748     foreach my $key (keys %$conf) {
2749         my $value = $conf->{$key};
2750         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2751             delete $conf->{$key};
2752         }
2753     }
2754
2755     my $generate_raw_config = sub {
2756         my ($conf, $pending) = @_;
2757
2758         my $raw = '';
2759
2760         # add description as comment to top of file
2761         if (defined(my $descr = $conf->{description})) {
2762             if ($descr) {
2763                 foreach my $cl (split(/\n/, $descr)) {
2764                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2765                 }
2766             } else {
2767                 $raw .= "#\n" if $pending;
2768             }
2769         }
2770
2771         foreach my $key (sort keys %$conf) {
2772             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2773             $raw .= "$key: $conf->{$key}\n";
2774         }
2775         return $raw;
2776     };
2777
2778     my $raw = &$generate_raw_config($conf);
2779
2780     if (scalar(keys %{$conf->{pending}})){
2781         $raw .= "\n[PENDING]\n";
2782         $raw .= &$generate_raw_config($conf->{pending}, 1);
2783     }
2784
2785     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2786         $raw .= "\n[$snapname]\n";
2787         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2788     }
2789
2790     return $raw;
2791 }
2792
2793 sub load_defaults {
2794
2795     my $res = {};
2796
2797     # we use static defaults from our JSON schema configuration
2798     foreach my $key (keys %$confdesc) {
2799         if (defined(my $default = $confdesc->{$key}->{default})) {
2800             $res->{$key} = $default;
2801         }
2802     }
2803
2804     return $res;
2805 }
2806
2807 sub config_list {
2808     my $vmlist = PVE::Cluster::get_vmlist();
2809     my $res = {};
2810     return $res if !$vmlist || !$vmlist->{ids};
2811     my $ids = $vmlist->{ids};
2812     my $nodename = nodename();
2813
2814     foreach my $vmid (keys %$ids) {
2815         my $d = $ids->{$vmid};
2816         next if !$d->{node} || $d->{node} ne $nodename;
2817         next if !$d->{type} || $d->{type} ne 'qemu';
2818         $res->{$vmid}->{exists} = 1;
2819     }
2820     return $res;
2821 }
2822
2823 # test if VM uses local resources (to prevent migration)
2824 sub check_local_resources {
2825     my ($conf, $noerr) = @_;
2826
2827     my @loc_res = ();
2828
2829     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2830     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2831
2832     push @loc_res, "ivshmem" if $conf->{ivshmem};
2833
2834     foreach my $k (keys %$conf) {
2835         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2836         # sockets are safe: they will recreated be on the target side post-migrate
2837         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2838         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2839     }
2840
2841     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2842
2843     return \@loc_res;
2844 }
2845
2846 # check if used storages are available on all nodes (use by migrate)
2847 sub check_storage_availability {
2848     my ($storecfg, $conf, $node) = @_;
2849
2850     foreach_drive($conf, sub {
2851         my ($ds, $drive) = @_;
2852
2853         my $volid = $drive->{file};
2854         return if !$volid;
2855
2856         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2857         return if !$sid;
2858
2859         # check if storage is available on both nodes
2860         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2861         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2862    });
2863 }
2864
2865 # list nodes where all VM images are available (used by has_feature API)
2866 sub shared_nodes {
2867     my ($conf, $storecfg) = @_;
2868
2869     my $nodelist = PVE::Cluster::get_nodelist();
2870     my $nodehash = { map { $_ => 1 } @$nodelist };
2871     my $nodename = nodename();
2872
2873     foreach_drive($conf, sub {
2874         my ($ds, $drive) = @_;
2875
2876         my $volid = $drive->{file};
2877         return if !$volid;
2878
2879         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2880         if ($storeid) {
2881             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2882             if ($scfg->{disable}) {
2883                 $nodehash = {};
2884             } elsif (my $avail = $scfg->{nodes}) {
2885                 foreach my $node (keys %$nodehash) {
2886                     delete $nodehash->{$node} if !$avail->{$node};
2887                 }
2888             } elsif (!$scfg->{shared}) {
2889                 foreach my $node (keys %$nodehash) {
2890                     delete $nodehash->{$node} if $node ne $nodename
2891                 }
2892             }
2893         }
2894     });
2895
2896     return $nodehash
2897 }
2898
2899 sub check_local_storage_availability {
2900     my ($conf, $storecfg) = @_;
2901
2902     my $nodelist = PVE::Cluster::get_nodelist();
2903     my $nodehash = { map { $_ => {} } @$nodelist };
2904
2905     foreach_drive($conf, sub {
2906         my ($ds, $drive) = @_;
2907
2908         my $volid = $drive->{file};
2909         return if !$volid;
2910
2911         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2912         if ($storeid) {
2913             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2914
2915             if ($scfg->{disable}) {
2916                 foreach my $node (keys %$nodehash) {
2917                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2918                 }
2919             } elsif (my $avail = $scfg->{nodes}) {
2920                 foreach my $node (keys %$nodehash) {
2921                     if (!$avail->{$node}) {
2922                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2923                     }
2924                 }
2925             }
2926         }
2927     });
2928
2929     foreach my $node (values %$nodehash) {
2930         if (my $unavail = $node->{unavailable_storages}) {
2931             $node->{unavailable_storages} = [ sort keys %$unavail ];
2932         }
2933     }
2934
2935     return $nodehash
2936 }
2937
2938 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2939 sub check_running {
2940     my ($vmid, $nocheck, $node) = @_;
2941
2942     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2943     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2944 }
2945
2946 sub vzlist {
2947
2948     my $vzlist = config_list();
2949
2950     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2951
2952     while (defined(my $de = $fd->read)) {
2953         next if $de !~ m/^(\d+)\.pid$/;
2954         my $vmid = $1;
2955         next if !defined($vzlist->{$vmid});
2956         if (my $pid = check_running($vmid)) {
2957             $vzlist->{$vmid}->{pid} = $pid;
2958         }
2959     }
2960
2961     return $vzlist;
2962 }
2963
2964 sub disksize {
2965     my ($storecfg, $conf) = @_;
2966
2967     my $bootdisk = $conf->{bootdisk};
2968     return undef if !$bootdisk;
2969     return undef if !is_valid_drivename($bootdisk);
2970
2971     return undef if !$conf->{$bootdisk};
2972
2973     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2974     return undef if !defined($drive);
2975
2976     return undef if drive_is_cdrom($drive);
2977
2978     my $volid = $drive->{file};
2979     return undef if !$volid;
2980
2981     return $drive->{size};
2982 }
2983
2984 our $vmstatus_return_properties = {
2985     vmid => get_standard_option('pve-vmid'),
2986     status => {
2987         description => "Qemu process status.",
2988         type => 'string',
2989         enum => ['stopped', 'running'],
2990     },
2991     maxmem => {
2992         description => "Maximum memory in bytes.",
2993         type => 'integer',
2994         optional => 1,
2995         renderer => 'bytes',
2996     },
2997     maxdisk => {
2998         description => "Root disk size in bytes.",
2999         type => 'integer',
3000         optional => 1,
3001         renderer => 'bytes',
3002     },
3003     name => {
3004         description => "VM name.",
3005         type => 'string',
3006         optional => 1,
3007     },
3008     qmpstatus => {
3009         description => "Qemu QMP agent status.",
3010         type => 'string',
3011         optional => 1,
3012     },
3013     pid => {
3014         description => "PID of running qemu process.",
3015         type => 'integer',
3016         optional => 1,
3017     },
3018     uptime => {
3019         description => "Uptime.",
3020         type => 'integer',
3021         optional => 1,
3022         renderer => 'duration',
3023     },
3024     cpus => {
3025         description => "Maximum usable CPUs.",
3026         type => 'number',
3027         optional => 1,
3028     },
3029     lock => {
3030         description => "The current config lock, if any.",
3031         type => 'string',
3032         optional => 1,
3033     },
3034     tags => {
3035         description  => "The current configured tags, if any",
3036         type => 'string',
3037         optional => 1,
3038     },
3039 };
3040
3041 my $last_proc_pid_stat;
3042
3043 # get VM status information
3044 # This must be fast and should not block ($full == false)
3045 # We only query KVM using QMP if $full == true (this can be slow)
3046 sub vmstatus {
3047     my ($opt_vmid, $full) = @_;
3048
3049     my $res = {};
3050
3051     my $storecfg = PVE::Storage::config();
3052
3053     my $list = vzlist();
3054     my $defaults = load_defaults();
3055
3056     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
3057
3058     my $cpucount = $cpuinfo->{cpus} || 1;
3059
3060     foreach my $vmid (keys %$list) {
3061         next if $opt_vmid && ($vmid ne $opt_vmid);
3062
3063         my $conf = PVE::QemuConfig->load_config($vmid);
3064
3065         my $d = { vmid => $vmid };
3066         $d->{pid} = $list->{$vmid}->{pid};
3067
3068         # fixme: better status?
3069         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3070
3071         my $size = disksize($storecfg, $conf);
3072         if (defined($size)) {
3073             $d->{disk} = 0; # no info available
3074             $d->{maxdisk} = $size;
3075         } else {
3076             $d->{disk} = 0;
3077             $d->{maxdisk} = 0;
3078         }
3079
3080         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3081             * ($conf->{cores} || $defaults->{cores});
3082         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3083         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3084
3085         $d->{name} = $conf->{name} || "VM $vmid";
3086         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3087             : $defaults->{memory}*(1024*1024);
3088
3089         if ($conf->{balloon}) {
3090             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3091             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3092                 : $defaults->{shares};
3093         }
3094
3095         $d->{uptime} = 0;
3096         $d->{cpu} = 0;
3097         $d->{mem} = 0;
3098
3099         $d->{netout} = 0;
3100         $d->{netin} = 0;
3101
3102         $d->{diskread} = 0;
3103         $d->{diskwrite} = 0;
3104
3105         $d->{template} = PVE::QemuConfig->is_template($conf);
3106
3107         $d->{serial} = 1 if conf_has_serial($conf);
3108         $d->{lock} = $conf->{lock} if $conf->{lock};
3109         $d->{tags} = $conf->{tags} if defined($conf->{tags});
3110
3111         $res->{$vmid} = $d;
3112     }
3113
3114     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3115     foreach my $dev (keys %$netdev) {
3116         next if $dev !~ m/^tap([1-9]\d*)i/;
3117         my $vmid = $1;
3118         my $d = $res->{$vmid};
3119         next if !$d;
3120
3121         $d->{netout} += $netdev->{$dev}->{receive};
3122         $d->{netin} += $netdev->{$dev}->{transmit};
3123
3124         if ($full) {
3125             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3126             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3127         }
3128
3129     }
3130
3131     my $ctime = gettimeofday;
3132
3133     foreach my $vmid (keys %$list) {
3134
3135         my $d = $res->{$vmid};
3136         my $pid = $d->{pid};
3137         next if !$pid;
3138
3139         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3140         next if !$pstat; # not running
3141
3142         my $used = $pstat->{utime} + $pstat->{stime};
3143
3144         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3145
3146         if ($pstat->{vsize}) {
3147             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3148         }
3149
3150         my $old = $last_proc_pid_stat->{$pid};
3151         if (!$old) {
3152             $last_proc_pid_stat->{$pid} = {
3153                 time => $ctime,
3154                 used => $used,
3155                 cpu => 0,
3156             };
3157             next;
3158         }
3159
3160         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3161
3162         if ($dtime > 1000) {
3163             my $dutime = $used -  $old->{used};
3164
3165             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3166             $last_proc_pid_stat->{$pid} = {
3167                 time => $ctime,
3168                 used => $used,
3169                 cpu => $d->{cpu},
3170             };
3171         } else {
3172             $d->{cpu} = $old->{cpu};
3173         }
3174     }
3175
3176     return $res if !$full;
3177
3178     my $qmpclient = PVE::QMPClient->new();
3179
3180     my $ballooncb = sub {
3181         my ($vmid, $resp) = @_;
3182
3183         my $info = $resp->{'return'};
3184         return if !$info->{max_mem};
3185
3186         my $d = $res->{$vmid};
3187
3188         # use memory assigned to VM
3189         $d->{maxmem} = $info->{max_mem};
3190         $d->{balloon} = $info->{actual};
3191
3192         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3193             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3194             $d->{freemem} = $info->{free_mem};
3195         }
3196
3197         $d->{ballooninfo} = $info;
3198     };
3199
3200     my $blockstatscb = sub {
3201         my ($vmid, $resp) = @_;
3202         my $data = $resp->{'return'} || [];
3203         my $totalrdbytes = 0;
3204         my $totalwrbytes = 0;
3205
3206         for my $blockstat (@$data) {
3207             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3208             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3209
3210             $blockstat->{device} =~ s/drive-//;
3211             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3212         }
3213         $res->{$vmid}->{diskread} = $totalrdbytes;
3214         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3215     };
3216
3217     my $statuscb = sub {
3218         my ($vmid, $resp) = @_;
3219
3220         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3221         # this fails if ballon driver is not loaded, so this must be
3222         # the last commnand (following command are aborted if this fails).
3223         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3224
3225         my $status = 'unknown';
3226         if (!defined($status = $resp->{'return'}->{status})) {
3227             warn "unable to get VM status\n";
3228             return;
3229         }
3230
3231         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3232     };
3233
3234     foreach my $vmid (keys %$list) {
3235         next if $opt_vmid && ($vmid ne $opt_vmid);
3236         next if !$res->{$vmid}->{pid}; # not running
3237         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3238     }
3239
3240     $qmpclient->queue_execute(undef, 2);
3241
3242     foreach my $vmid (keys %$list) {
3243         next if $opt_vmid && ($vmid ne $opt_vmid);
3244         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3245     }
3246
3247     return $res;
3248 }
3249
3250 sub foreach_drive {
3251     my ($conf, $func, @param) = @_;
3252
3253     foreach my $ds (valid_drive_names()) {
3254         next if !defined($conf->{$ds});
3255
3256         my $drive = parse_drive($ds, $conf->{$ds});
3257         next if !$drive;
3258
3259         &$func($ds, $drive, @param);
3260     }
3261 }
3262
3263 sub foreach_volid {
3264     my ($conf, $func, @param) = @_;
3265
3266     my $volhash = {};
3267
3268     my $test_volid = sub {
3269         my ($volid, $is_cdrom, $replicate, $shared, $snapname, $size) = @_;
3270
3271         return if !$volid;
3272
3273         $volhash->{$volid}->{cdrom} //= 1;
3274         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3275
3276         $volhash->{$volid}->{replicate} //= 0;
3277         $volhash->{$volid}->{replicate} = 1 if $replicate;
3278
3279         $volhash->{$volid}->{shared} //= 0;
3280         $volhash->{$volid}->{shared} = 1 if $shared;
3281
3282         $volhash->{$volid}->{referenced_in_config} //= 0;
3283         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3284
3285         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3286             if defined($snapname);
3287         $volhash->{$volid}->{size} = $size if $size;
3288     };
3289
3290     foreach_drive($conf, sub {
3291         my ($ds, $drive) = @_;
3292         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef, $drive->{size});
3293     });
3294
3295     foreach my $snapname (keys %{$conf->{snapshots}}) {
3296         my $snap = $conf->{snapshots}->{$snapname};
3297         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3298         foreach_drive($snap, sub {
3299             my ($ds, $drive) = @_;
3300             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3301         });
3302     }
3303
3304     foreach my $volid (keys %$volhash) {
3305         &$func($volid, $volhash->{$volid}, @param);
3306     }
3307 }
3308
3309 sub conf_has_serial {
3310     my ($conf) = @_;
3311
3312     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3313         if ($conf->{"serial$i"}) {
3314             return 1;
3315         }
3316     }
3317
3318     return 0;
3319 }
3320
3321 sub conf_has_audio {
3322     my ($conf, $id) = @_;
3323
3324     $id //= 0;
3325     my $audio = $conf->{"audio$id"};
3326     return undef if !defined($audio);
3327
3328     my $audioproperties = PVE::JSONSchema::parse_property_string($audio_fmt, $audio);
3329     my $audiodriver = $audioproperties->{driver} // 'spice';
3330
3331     return {
3332         dev => $audioproperties->{device},
3333         dev_id => "audiodev$id",
3334         backend => $audiodriver,
3335         backend_id => "$audiodriver-backend${id}",
3336     };
3337 }
3338
3339 sub vga_conf_has_spice {
3340     my ($vga) = @_;
3341
3342     my $vgaconf = parse_vga($vga);
3343     my $vgatype = $vgaconf->{type};
3344     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3345
3346     return $1 || 1;
3347 }
3348
3349 sub is_native($) {
3350     my ($arch) = @_;
3351     return get_host_arch() eq $arch;
3352 }
3353
3354 sub get_vm_arch {
3355     my ($conf) = @_;
3356     return $conf->{arch} // get_host_arch();
3357 }
3358
3359 my $default_machines = {
3360     x86_64 => 'pc',
3361     aarch64 => 'virt',
3362 };
3363
3364 sub get_vm_machine {
3365     my ($conf, $forcemachine, $arch, $add_pve_version) = @_;
3366
3367     my $machine = $forcemachine || $conf->{machine};
3368
3369     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3370         $arch //= 'x86_64';
3371         $machine ||= $default_machines->{$arch};
3372         $machine .= "+pve$PVE::QemuServer::Machine::PVE_MACHINE_VERSION" if $add_pve_version;
3373     }
3374
3375     return $machine;
3376 }
3377
3378 sub get_ovmf_files($) {
3379     my ($arch) = @_;
3380
3381     my $ovmf = $OVMF->{$arch}
3382         or die "no OVMF images known for architecture '$arch'\n";
3383
3384     return @$ovmf;
3385 }
3386
3387 my $Arch2Qemu = {
3388     aarch64 => '/usr/bin/qemu-system-aarch64',
3389     x86_64 => '/usr/bin/qemu-system-x86_64',
3390 };
3391 sub get_command_for_arch($) {
3392     my ($arch) = @_;
3393     return '/usr/bin/kvm' if is_native($arch);
3394
3395     my $cmd = $Arch2Qemu->{$arch}
3396         or die "don't know how to emulate architecture '$arch'\n";
3397     return $cmd;
3398 }
3399
3400 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3401 # to use in a QEMU command line (-cpu element), first array_intersect the result
3402 # of query_supported_ with query_understood_. This is necessary because:
3403 #
3404 # a) query_understood_ returns flags the host cannot use and
3405 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3406 #    flags, but CPU settings - with most of them being flags. Those settings
3407 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3408 #
3409 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3410 # expensive. If you need the value returned from this, you can get it much
3411 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3412 # $accel being 'kvm' or 'tcg'.
3413 #
3414 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3415 # changes, automatically populating pmxcfs.
3416 #
3417 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3418 # since kvm and tcg machines support different flags
3419 #
3420 sub query_supported_cpu_flags {
3421     my $flags = {};
3422
3423     my ($arch, $default_machine) = get_basic_machine_info();
3424
3425     # FIXME: Once this is merged, the code below should work for ARM as well:
3426     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3427     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3428         $arch eq "aarch64";
3429
3430     my $kvm_supported = defined(kvm_version());
3431     my $qemu_cmd = get_command_for_arch($arch);
3432     my $fakevmid = -1;
3433     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3434
3435     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3436     my $query_supported_run_qemu = sub {
3437         my ($kvm) = @_;
3438
3439         my $flags = {};
3440         my $cmd = [
3441             $qemu_cmd,
3442             '-machine', $default_machine,
3443             '-display', 'none',
3444             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server,nowait",
3445             '-mon', 'chardev=qmp,mode=control',
3446             '-pidfile', $pidfile,
3447             '-S', '-daemonize'
3448         ];
3449
3450         if (!$kvm) {
3451             push @$cmd, '-accel', 'tcg';
3452         }
3453
3454         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3455         die "QEMU flag querying VM exited with code " . $rc if $rc;
3456
3457         eval {
3458             my $cmd_result = mon_cmd(
3459                 $fakevmid,
3460                 'query-cpu-model-expansion',
3461                 type => 'full',
3462                 model => { name => 'host' }
3463             );
3464
3465             my $props = $cmd_result->{model}->{props};
3466             foreach my $prop (keys %$props) {
3467                 next if $props->{$prop} ne '1';
3468                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3469                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3470                 # We only keep those with underscores, to match /proc/cpuinfo
3471                 $prop =~ s/\.|-/_/g;
3472                 $flags->{$prop} = 1;
3473             }
3474         };
3475         my $err = $@;
3476
3477         # force stop with 10 sec timeout and 'nocheck'
3478         # always stop, even if QMP failed
3479         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3480
3481         die $err if $err;
3482
3483         return [ sort keys %$flags ];
3484     };
3485
3486     # We need to query QEMU twice, since KVM and TCG have different supported flags
3487     PVE::QemuConfig->lock_config($fakevmid, sub {
3488         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3489         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3490
3491         if ($kvm_supported) {
3492             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3493             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3494         }
3495     });
3496
3497     return $flags;
3498 }
3499
3500 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3501 my $understood_cpu_flag_dir = "/usr/share/kvm";
3502 sub query_understood_cpu_flags {
3503     my $arch = get_host_arch();
3504     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3505
3506     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3507         if ! -e $filepath;
3508
3509     my $raw = file_get_contents($filepath);
3510     $raw =~ s/^\s+|\s+$//g;
3511     my @flags = split(/\s+/, $raw);
3512
3513     return \@flags;
3514 }
3515
3516 sub get_cpu_options {
3517     my ($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough) = @_;
3518
3519     my $cpuFlags = [];
3520     my $ostype = $conf->{ostype};
3521
3522     my $cpu = $kvm ? "kvm64" : "qemu64";
3523     if ($arch eq 'aarch64') {
3524         $cpu = 'cortex-a57';
3525     }
3526     my $hv_vendor_id;
3527     if (my $cputype = $conf->{cpu}) {
3528         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3529             or die "Cannot parse cpu description: $cputype\n";
3530         $cpu = $cpuconf->{cputype};
3531         $kvm_off = 1 if $cpuconf->{hidden};
3532         $hv_vendor_id = $cpuconf->{'hv-vendor-id'};
3533
3534         if (defined(my $flags = $cpuconf->{flags})) {
3535             push @$cpuFlags, split(";", $flags);
3536         }
3537     }
3538
3539     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3540
3541     push @$cpuFlags , '-x2apic' if $ostype && $ostype eq 'solaris';
3542
3543     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3544
3545     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3546
3547     if (min_version($machine_version, 2, 3) && $arch eq 'x86_64') {
3548
3549         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3550         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3551     }
3552
3553     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_version, $conf->{bios}, $gpu_passthrough, $hv_vendor_id) if $kvm;
3554
3555     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3556
3557     push @$cpuFlags, 'kvm=off' if $kvm_off;
3558
3559     if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3560         push @$cpuFlags, "vendor=${cpu_vendor}"
3561             if $cpu_vendor ne 'default';
3562     } elsif ($arch ne 'aarch64') {
3563         die "internal error"; # should not happen
3564     }
3565
3566     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3567
3568     return ('-cpu', $cpu);
3569 }
3570
3571 sub config_to_command {
3572     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3573
3574     my $cmd = [];
3575     my $globalFlags = [];
3576     my $machineFlags = [];
3577     my $rtcFlags = [];
3578     my $devices = [];
3579     my $pciaddr = '';
3580     my $bridges = {};
3581     my $vernum = 0; # unknown
3582     my $ostype = $conf->{ostype};
3583     my $winversion = windows_version($ostype);
3584     my $kvm = $conf->{kvm};
3585     my $nodename = nodename();
3586
3587     my $arch = get_vm_arch($conf);
3588     my $kvm_binary = get_command_for_arch($arch);
3589     my $kvmver = kvm_user_version($kvm_binary);
3590
3591     my $add_pve_version = min_version($kvmver, 4, 1);
3592
3593     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3594     my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type, $kvmver);
3595     $kvm //= 1 if is_native($arch);
3596
3597     $machine_version =~ m/(\d+)\.(\d+)/;
3598     die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type', please upgrade node '$nodename'\n"
3599         if !PVE::QemuServer::min_version($kvmver, $1, $2);
3600
3601     if ($kvm) {
3602         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3603             if !defined kvm_version();
3604     }
3605
3606     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3607         $vernum = $1*1000000+$2*1000;
3608     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3609         $vernum = $1*1000000+$2*1000+$3;
3610     }
3611
3612     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3613
3614     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3615     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3616     my $use_old_bios_files = undef;
3617     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3618
3619     my $cpuunits = defined($conf->{cpuunits}) ?
3620             $conf->{cpuunits} : $defaults->{cpuunits};
3621
3622     push @$cmd, $kvm_binary;
3623
3624     push @$cmd, '-id', $vmid;
3625
3626     my $vmname = $conf->{name} || "vm$vmid";
3627
3628     push @$cmd, '-name', $vmname;
3629
3630     my $use_virtio = 0;
3631
3632     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3633     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3634     push @$cmd, '-mon', "chardev=qmp,mode=control";
3635
3636     if (min_version($machine_version, 2, 12)) {
3637         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3638         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3639     }
3640
3641     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3642
3643     push @$cmd, '-daemonize';
3644
3645     if ($conf->{smbios1}) {
3646         my $smbios_conf = parse_smbios1($conf->{smbios1});
3647         if ($smbios_conf->{base64}) {
3648             # Do not pass base64 flag to qemu
3649             delete $smbios_conf->{base64};
3650             my $smbios_string = "";
3651             foreach my $key (keys %$smbios_conf) {
3652                 my $value;
3653                 if ($key eq "uuid") {
3654                     $value = $smbios_conf->{uuid}
3655                 } else {
3656                     $value = decode_base64($smbios_conf->{$key});
3657                 }
3658                 # qemu accepts any binary data, only commas need escaping by double comma
3659                 $value =~ s/,/,,/g;
3660                 $smbios_string .= "," . $key . "=" . $value if $value;
3661             }
3662             push @$cmd, '-smbios', "type=1" . $smbios_string;
3663         } else {
3664             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3665         }
3666     }
3667
3668     if ($conf->{vmgenid}) {
3669         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3670     }
3671
3672     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3673     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3674         die "uefi base image not found\n" if ! -f $ovmf_code;
3675
3676         my $path;
3677         my $format;
3678         if (my $efidisk = $conf->{efidisk0}) {
3679             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3680             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3681             $format = $d->{format};
3682             if ($storeid) {
3683                 $path = PVE::Storage::path($storecfg, $d->{file});
3684                 if (!defined($format)) {
3685                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3686                     $format = qemu_img_format($scfg, $volname);
3687                 }
3688             } else {
3689                 $path = $d->{file};
3690                 die "efidisk format must be specified\n"
3691                     if !defined($format);
3692             }
3693         } else {
3694             warn "no efidisk configured! Using temporary efivars disk.\n";
3695             $path = "/tmp/$vmid-ovmf.fd";
3696             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3697             $format = 'raw';
3698         }
3699
3700         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3701         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3702     }
3703
3704     # load q35 config
3705     if ($q35) {
3706         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3707         if (min_version($machine_version, 4, 0)) {
3708             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3709         } else {
3710             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3711         }
3712     }
3713
3714     # add usb controllers
3715     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3716     push @$devices, @usbcontrollers if @usbcontrollers;
3717     my $vga = parse_vga($conf->{vga});
3718
3719     my $qxlnum = vga_conf_has_spice($conf->{vga});
3720     $vga->{type} = 'qxl' if $qxlnum;
3721
3722     if (!$vga->{type}) {
3723         if ($arch eq 'aarch64') {
3724             $vga->{type} = 'virtio';
3725         } elsif (min_version($machine_version, 2, 9)) {
3726             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3727         } else {
3728             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3729         }
3730     }
3731
3732     # enable absolute mouse coordinates (needed by vnc)
3733     my $tablet;
3734     if (defined($conf->{tablet})) {
3735         $tablet = $conf->{tablet};
3736     } else {
3737         $tablet = $defaults->{tablet};
3738         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3739         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3740     }
3741
3742     if ($tablet) {
3743         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3744         my $kbd = print_keyboarddevice_full($conf, $arch);
3745         push @$devices, '-device', $kbd if defined($kbd);
3746     }
3747
3748     my $kvm_off = 0;
3749     my $gpu_passthrough;
3750
3751     # host pci devices
3752     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3753         my $id = "hostpci$i";
3754         my $d = parse_hostpci($conf->{$id});
3755         next if !$d;
3756
3757         if (my $pcie = $d->{pcie}) {
3758             die "q35 machine model is not enabled" if !$q35;
3759             # win7 wants to have the pcie devices directly on the pcie bus
3760             # instead of in the root port
3761             if ($winversion == 7) {
3762                 $pciaddr = print_pcie_addr("${id}bus0");
3763             } else {
3764                 # add more root ports if needed, 4 are present by default
3765                 # by pve-q35 cfgs, rest added here on demand.
3766                 if ($i > 3) {
3767                     push @$devices, '-device', print_pcie_root_port($i);
3768                 }
3769                 $pciaddr = print_pcie_addr($id);
3770             }
3771         } else {
3772             $pciaddr = print_pci_addr($id, $bridges, $arch, $machine_type);
3773         }
3774
3775         my $xvga = '';
3776         if ($d->{'x-vga'}) {
3777             $xvga = ',x-vga=on' if !($conf->{bios} && $conf->{bios} eq 'ovmf');
3778             $kvm_off = 1;
3779             $vga->{type} = 'none' if !defined($conf->{vga});
3780             $gpu_passthrough = 1;
3781         }
3782
3783         my $pcidevices = $d->{pciid};
3784         my $multifunction = 1 if @$pcidevices > 1;
3785
3786         my $sysfspath;
3787         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3788             my $pci_id = $pcidevices->[0]->{id};
3789             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3790             $sysfspath = "/sys/bus/pci/devices/$pci_id/$uuid";
3791         } elsif ($d->{mdev}) {
3792             warn "ignoring mediated device '$id' with multifunction device\n";
3793         }
3794
3795         my $j=0;
3796         foreach my $pcidevice (@$pcidevices) {
3797             my $devicestr = "vfio-pci";
3798
3799             if ($sysfspath) {
3800                 $devicestr .= ",sysfsdev=$sysfspath";
3801             } else {
3802                 $devicestr .= ",host=$pcidevice->{id}";
3803             }
3804
3805             my $mf_addr = $multifunction ? ".$j" : '';
3806             $devicestr .= ",id=${id}${mf_addr}${pciaddr}${mf_addr}";
3807
3808             if ($j == 0) {
3809                 $devicestr .= ',rombar=0' if defined($d->{rombar}) && !$d->{rombar};
3810                 $devicestr .= "$xvga";
3811                 $devicestr .= ",multifunction=on" if $multifunction;
3812                 $devicestr .= ",romfile=/usr/share/kvm/$d->{romfile}" if $d->{romfile};
3813             }
3814
3815             push @$devices, '-device', $devicestr;
3816             $j++;
3817         }
3818     }
3819
3820     # usb devices
3821     my $usb_dev_features = {};
3822     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3823
3824     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features);
3825     push @$devices, @usbdevices if @usbdevices;
3826     # serial devices
3827     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3828         if (my $path = $conf->{"serial$i"}) {
3829             if ($path eq 'socket') {
3830                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3831                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3832                 # On aarch64, serial0 is the UART device. Qemu only allows
3833                 # connecting UART devices via the '-serial' command line, as
3834                 # the device has a fixed slot on the hardware...
3835                 if ($arch eq 'aarch64' && $i == 0) {
3836                     push @$devices, '-serial', "chardev:serial$i";
3837                 } else {
3838                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3839                 }
3840             } else {
3841                 die "no such serial device\n" if ! -c $path;
3842                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3843                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3844             }
3845         }
3846     }
3847
3848     # parallel devices
3849     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3850         if (my $path = $conf->{"parallel$i"}) {
3851             die "no such parallel device\n" if ! -c $path;
3852             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3853             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3854             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3855         }
3856     }
3857
3858     if (my $audio = conf_has_audio($conf)) {
3859
3860         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3861
3862         my $id = $audio->{dev_id};
3863         if ($audio->{dev} eq 'AC97') {
3864             push @$devices, '-device', "AC97,id=${id}${audiopciaddr}";
3865         } elsif ($audio->{dev} =~ /intel\-hda$/) {
3866             push @$devices, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3867             push @$devices, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0";
3868             push @$devices, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1";
3869         } else {
3870             die "unkown audio device '$audio->{dev}', implement me!";
3871         }
3872
3873         push @$devices, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3874     }
3875
3876     my $sockets = 1;
3877     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3878     $sockets = $conf->{sockets} if  $conf->{sockets};
3879
3880     my $cores = $conf->{cores} || 1;
3881
3882     my $maxcpus = $sockets * $cores;
3883
3884     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3885
3886     my $allowed_vcpus = $cpuinfo->{cpus};
3887
3888     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3889         if ($allowed_vcpus < $maxcpus);
3890
3891     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3892
3893         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3894         for (my $i = 2; $i <= $vcpus; $i++)  {
3895             my $cpustr = print_cpu_device($conf,$i);
3896             push @$cmd, '-device', $cpustr;
3897         }
3898
3899     } else {
3900
3901         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3902     }
3903     push @$cmd, '-nodefaults';
3904
3905     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3906
3907     my $bootindex_hash = {};
3908     my $i = 1;
3909     foreach my $o (split(//, $bootorder)) {
3910         $bootindex_hash->{$o} = $i*100;
3911         $i++;
3912     }
3913
3914     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3915
3916     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3917
3918     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3919
3920     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3921         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3922         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3923         push @$cmd,  '-vnc', "unix:$socket,password";
3924     } else {
3925         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3926         push @$cmd, '-nographic';
3927     }
3928
3929     # time drift fix
3930     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3931
3932     my $useLocaltime = $conf->{localtime};
3933
3934     if ($winversion >= 5) { # windows
3935         $useLocaltime = 1 if !defined($conf->{localtime});
3936
3937         # use time drift fix when acpi is enabled
3938         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3939             $tdf = 1 if !defined($conf->{tdf});
3940         }
3941     }
3942
3943     if ($winversion >= 6) {
3944         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3945         push @$cmd, '-no-hpet';
3946     }
3947
3948     push @$rtcFlags, 'driftfix=slew' if $tdf;
3949
3950     if (!$kvm) {
3951         push @$machineFlags, 'accel=tcg';
3952     }
3953
3954     if ($machine_type) {
3955         push @$machineFlags, "type=${machine_type}";
3956     }
3957
3958     if (($conf->{startdate}) && ($conf->{startdate} ne 'now')) {
3959         push @$rtcFlags, "base=$conf->{startdate}";
3960     } elsif ($useLocaltime) {
3961         push @$rtcFlags, 'base=localtime';
3962     }
3963
3964     push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3965
3966     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3967
3968     push @$cmd, '-S' if $conf->{freeze};
3969
3970     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3971
3972     my $guest_agent = parse_guest_agent($conf);
3973
3974     if ($guest_agent->{enabled}) {
3975         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3976         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3977
3978         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3979             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3980             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3981             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3982         } elsif ($guest_agent->{type} eq 'isa') {
3983             push @$devices, '-device', "isa-serial,chardev=qga0";
3984         }
3985     }
3986
3987     my $spice_port;
3988
3989     if ($qxlnum) {
3990         if ($qxlnum > 1) {
3991             if ($winversion){
3992                 for(my $i = 1; $i < $qxlnum; $i++){
3993                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3994                 }
3995             } else {
3996                 # assume other OS works like Linux
3997                 my ($ram, $vram) = ("134217728", "67108864");
3998                 if ($vga->{memory}) {
3999                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
4000                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
4001                 }
4002                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
4003                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
4004             }
4005         }
4006
4007         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
4008
4009         my $pfamily = PVE::Tools::get_host_address_family($nodename);
4010         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
4011         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
4012
4013         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
4014         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
4015         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
4016
4017         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
4018         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
4019
4020         my $spice_enhancement = PVE::JSONSchema::parse_property_string($spice_enhancements_fmt, $conf->{spice_enhancements} // '');
4021         if ($spice_enhancement->{foldersharing}) {
4022             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
4023             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
4024         }
4025
4026         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
4027         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming};
4028         push @$devices, '-spice', "$spice_opts";
4029     }
4030
4031     # enable balloon by default, unless explicitly disabled
4032     if (!defined($conf->{balloon}) || $conf->{balloon}) {
4033         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
4034         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
4035     }
4036
4037     if ($conf->{watchdog}) {
4038         my $wdopts = parse_watchdog($conf->{watchdog});
4039         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
4040         my $watchdog = $wdopts->{model} || 'i6300esb';
4041         push @$devices, '-device', "$watchdog$pciaddr";
4042         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
4043     }
4044
4045     my $vollist = [];
4046     my $scsicontroller = {};
4047     my $ahcicontroller = {};
4048     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
4049
4050     # Add iscsi initiator name if available
4051     if (my $initiator = get_initiator_name()) {
4052         push @$devices, '-iscsi', "initiator-name=$initiator";
4053     }
4054
4055     foreach_drive($conf, sub {
4056         my ($ds, $drive) = @_;
4057
4058         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
4059             push @$vollist, $drive->{file};
4060         }
4061
4062         # ignore efidisk here, already added in bios/fw handling code above
4063         return if $drive->{interface} eq 'efidisk';
4064
4065         $use_virtio = 1 if $ds =~ m/^virtio/;
4066
4067         if (drive_is_cdrom ($drive)) {
4068             if ($bootindex_hash->{d}) {
4069                 $drive->{bootindex} = $bootindex_hash->{d};
4070                 $bootindex_hash->{d} += 1;
4071             }
4072         } else {
4073             if ($bootindex_hash->{c}) {
4074                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
4075                 $bootindex_hash->{c} += 1;
4076             }
4077         }
4078
4079         if($drive->{interface} eq 'virtio'){
4080            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
4081         }
4082
4083         if ($drive->{interface} eq 'scsi') {
4084
4085             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
4086
4087             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
4088             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
4089
4090             my $iothread = '';
4091             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
4092                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
4093                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
4094             } elsif ($drive->{iothread}) {
4095                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
4096             }
4097
4098             my $queues = '';
4099             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
4100                 $queues = ",num_queues=$drive->{queues}";
4101             }
4102
4103             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
4104             $scsicontroller->{$controller}=1;
4105         }
4106
4107         if ($drive->{interface} eq 'sata') {
4108            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
4109            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
4110            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
4111            $ahcicontroller->{$controller}=1;
4112         }
4113
4114         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
4115         push @$devices, '-drive',$drive_cmd;
4116         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4117     });
4118
4119     for (my $i = 0; $i < $MAX_NETS; $i++) {
4120          next if !$conf->{"net$i"};
4121          my $d = parse_net($conf->{"net$i"});
4122          next if !$d;
4123
4124          $use_virtio = 1 if $d->{model} eq 'virtio';
4125
4126          if ($bootindex_hash->{n}) {
4127             $d->{bootindex} = $bootindex_hash->{n};
4128             $bootindex_hash->{n} += 1;
4129          }
4130
4131          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
4132          push @$devices, '-netdev', $netdevfull;
4133
4134          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
4135          push @$devices, '-device', $netdevicefull;
4136     }
4137
4138     if ($conf->{ivshmem}) {
4139         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4140
4141         my $bus;
4142         if ($q35) {
4143             $bus = print_pcie_addr("ivshmem");
4144         } else {
4145             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4146         }
4147
4148         my $ivshmem_name = $ivshmem->{name} // $vmid;
4149         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4150
4151         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4152         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
4153     }
4154
4155     if (!$q35) {
4156         # add pci bridges
4157         if (min_version($machine_version, 2, 3)) {
4158            $bridges->{1} = 1;
4159            $bridges->{2} = 1;
4160         }
4161
4162         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4163
4164         for my $k (sort {$b cmp $a} keys %$bridges) {
4165             $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
4166             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
4167         }
4168     }
4169
4170     push @$cmd, @$devices;
4171     push @$cmd, '-rtc', join(',', @$rtcFlags)
4172         if scalar(@$rtcFlags);
4173     push @$cmd, '-machine', join(',', @$machineFlags)
4174         if scalar(@$machineFlags);
4175     push @$cmd, '-global', join(',', @$globalFlags)
4176         if scalar(@$globalFlags);
4177
4178     if (my $vmstate = $conf->{vmstate}) {
4179         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4180         push @$vollist, $vmstate;
4181         push @$cmd, '-loadstate', $statepath;
4182         print "activating and using '$vmstate' as vmstate\n";
4183     }
4184
4185     # add custom args
4186     if ($conf->{args}) {
4187         my $aa = PVE::Tools::split_args($conf->{args});
4188         push @$cmd, @$aa;
4189     }
4190
4191     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4192 }
4193
4194 sub spice_port {
4195     my ($vmid) = @_;
4196
4197     my $res = mon_cmd($vmid, 'query-spice');
4198
4199     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4200 }
4201
4202 sub vm_devices_list {
4203     my ($vmid) = @_;
4204
4205     my $res = mon_cmd($vmid, 'query-pci');
4206     my $devices_to_check = [];
4207     my $devices = {};
4208     foreach my $pcibus (@$res) {
4209         push @$devices_to_check, @{$pcibus->{devices}},
4210     }
4211
4212     while (@$devices_to_check) {
4213         my $to_check = [];
4214         for my $d (@$devices_to_check) {
4215             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4216             next if !$d->{'pci_bridge'};
4217
4218             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4219             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4220         }
4221         $devices_to_check = $to_check;
4222     }
4223
4224     my $resblock = mon_cmd($vmid, 'query-block');
4225     foreach my $block (@$resblock) {
4226         if($block->{device} =~ m/^drive-(\S+)/){
4227                 $devices->{$1} = 1;
4228         }
4229     }
4230
4231     my $resmice = mon_cmd($vmid, 'query-mice');
4232     foreach my $mice (@$resmice) {
4233         if ($mice->{name} eq 'QEMU HID Tablet') {
4234             $devices->{tablet} = 1;
4235             last;
4236         }
4237     }
4238
4239     # for usb devices there is no query-usb
4240     # but we can iterate over the entries in
4241     # qom-list path=/machine/peripheral
4242     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4243     foreach my $per (@$resperipheral) {
4244         if ($per->{name} =~ m/^usb\d+$/) {
4245             $devices->{$per->{name}} = 1;
4246         }
4247     }
4248
4249     return $devices;
4250 }
4251
4252 sub vm_deviceplug {
4253     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4254
4255     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4256
4257     my $devices_list = vm_devices_list($vmid);
4258     return 1 if defined($devices_list->{$deviceid});
4259
4260     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
4261
4262     if ($deviceid eq 'tablet') {
4263
4264         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4265
4266     } elsif ($deviceid eq 'keyboard') {
4267
4268         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4269
4270     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4271
4272         die "usb hotplug currently not reliable\n";
4273         # since we can't reliably hot unplug all added usb devices
4274         # and usb passthrough disables live migration
4275         # we disable usb hotplugging for now
4276         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4277
4278     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4279
4280         qemu_iothread_add($vmid, $deviceid, $device);
4281
4282         qemu_driveadd($storecfg, $vmid, $device);
4283         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4284
4285         qemu_deviceadd($vmid, $devicefull);
4286         eval { qemu_deviceaddverify($vmid, $deviceid); };
4287         if (my $err = $@) {
4288             eval { qemu_drivedel($vmid, $deviceid); };
4289             warn $@ if $@;
4290             die $err;
4291         }
4292
4293     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4294
4295
4296         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4297         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4298         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4299
4300         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4301
4302         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4303             qemu_iothread_add($vmid, $deviceid, $device);
4304             $devicefull .= ",iothread=iothread-$deviceid";
4305         }
4306
4307         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4308             $devicefull .= ",num_queues=$device->{queues}";
4309         }
4310
4311         qemu_deviceadd($vmid, $devicefull);
4312         qemu_deviceaddverify($vmid, $deviceid);
4313
4314     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4315
4316         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4317         qemu_driveadd($storecfg, $vmid, $device);
4318
4319         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4320         eval { qemu_deviceadd($vmid, $devicefull); };
4321         if (my $err = $@) {
4322             eval { qemu_drivedel($vmid, $deviceid); };
4323             warn $@ if $@;
4324             die $err;
4325         }
4326
4327     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4328
4329         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4330
4331         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4332         my $use_old_bios_files = undef;
4333         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4334
4335         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4336         qemu_deviceadd($vmid, $netdevicefull);
4337         eval {
4338             qemu_deviceaddverify($vmid, $deviceid);
4339             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4340         };
4341         if (my $err = $@) {
4342             eval { qemu_netdevdel($vmid, $deviceid); };
4343             warn $@ if $@;
4344             die $err;
4345         }
4346
4347     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4348
4349         my $bridgeid = $2;
4350         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4351         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4352
4353         qemu_deviceadd($vmid, $devicefull);
4354         qemu_deviceaddverify($vmid, $deviceid);
4355
4356     } else {
4357         die "can't hotplug device '$deviceid'\n";
4358     }
4359
4360     return 1;
4361 }
4362
4363 # fixme: this should raise exceptions on error!
4364 sub vm_deviceunplug {
4365     my ($vmid, $conf, $deviceid) = @_;
4366
4367     my $devices_list = vm_devices_list($vmid);
4368     return 1 if !defined($devices_list->{$deviceid});
4369
4370     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4371
4372     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4373
4374         qemu_devicedel($vmid, $deviceid);
4375
4376     } elsif ($deviceid =~ m/^usb\d+$/) {
4377
4378         die "usb hotplug currently not reliable\n";
4379         # when unplugging usb devices this way,
4380         # there may be remaining usb controllers/hubs
4381         # so we disable it for now
4382         qemu_devicedel($vmid, $deviceid);
4383         qemu_devicedelverify($vmid, $deviceid);
4384
4385     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4386
4387         qemu_devicedel($vmid, $deviceid);
4388         qemu_devicedelverify($vmid, $deviceid);
4389         qemu_drivedel($vmid, $deviceid);
4390         qemu_iothread_del($conf, $vmid, $deviceid);
4391
4392     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4393
4394         qemu_devicedel($vmid, $deviceid);
4395         qemu_devicedelverify($vmid, $deviceid);
4396         qemu_iothread_del($conf, $vmid, $deviceid);
4397
4398     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4399
4400         qemu_devicedel($vmid, $deviceid);
4401         qemu_drivedel($vmid, $deviceid);
4402         qemu_deletescsihw($conf, $vmid, $deviceid);
4403
4404     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4405
4406         qemu_devicedel($vmid, $deviceid);
4407         qemu_devicedelverify($vmid, $deviceid);
4408         qemu_netdevdel($vmid, $deviceid);
4409
4410     } else {
4411         die "can't unplug device '$deviceid'\n";
4412     }
4413
4414     return 1;
4415 }
4416
4417 sub qemu_deviceadd {
4418     my ($vmid, $devicefull) = @_;
4419
4420     $devicefull = "driver=".$devicefull;
4421     my %options =  split(/[=,]/, $devicefull);
4422
4423     mon_cmd($vmid, "device_add" , %options);
4424 }
4425
4426 sub qemu_devicedel {
4427     my ($vmid, $deviceid) = @_;
4428
4429     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4430 }
4431
4432 sub qemu_iothread_add {
4433     my($vmid, $deviceid, $device) = @_;
4434
4435     if ($device->{iothread}) {
4436         my $iothreads = vm_iothreads_list($vmid);
4437         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4438     }
4439 }
4440
4441 sub qemu_iothread_del {
4442     my($conf, $vmid, $deviceid) = @_;
4443
4444     my $confid = $deviceid;
4445     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4446         $confid = 'scsi' . $1;
4447     }
4448     my $device = parse_drive($confid, $conf->{$confid});
4449     if ($device->{iothread}) {
4450         my $iothreads = vm_iothreads_list($vmid);
4451         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4452     }
4453 }
4454
4455 sub qemu_objectadd {
4456     my($vmid, $objectid, $qomtype) = @_;
4457
4458     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4459
4460     return 1;
4461 }
4462
4463 sub qemu_objectdel {
4464     my($vmid, $objectid) = @_;
4465
4466     mon_cmd($vmid, "object-del", id => $objectid);
4467
4468     return 1;
4469 }
4470
4471 sub qemu_driveadd {
4472     my ($storecfg, $vmid, $device) = @_;
4473
4474     my $drive = print_drive_full($storecfg, $vmid, $device);
4475     $drive =~ s/\\/\\\\/g;
4476     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4477
4478     # If the command succeeds qemu prints: "OK"
4479     return 1 if $ret =~ m/OK/s;
4480
4481     die "adding drive failed: $ret\n";
4482 }
4483
4484 sub qemu_drivedel {
4485     my($vmid, $deviceid) = @_;
4486
4487     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4488     $ret =~ s/^\s+//;
4489
4490     return 1 if $ret eq "";
4491
4492     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4493     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4494
4495     die "deleting drive $deviceid failed : $ret\n";
4496 }
4497
4498 sub qemu_deviceaddverify {
4499     my ($vmid, $deviceid) = @_;
4500
4501     for (my $i = 0; $i <= 5; $i++) {
4502          my $devices_list = vm_devices_list($vmid);
4503          return 1 if defined($devices_list->{$deviceid});
4504          sleep 1;
4505     }
4506
4507     die "error on hotplug device '$deviceid'\n";
4508 }
4509
4510
4511 sub qemu_devicedelverify {
4512     my ($vmid, $deviceid) = @_;
4513
4514     # need to verify that the device is correctly removed as device_del
4515     # is async and empty return is not reliable
4516
4517     for (my $i = 0; $i <= 5; $i++) {
4518          my $devices_list = vm_devices_list($vmid);
4519          return 1 if !defined($devices_list->{$deviceid});
4520          sleep 1;
4521     }
4522
4523     die "error on hot-unplugging device '$deviceid'\n";
4524 }
4525
4526 sub qemu_findorcreatescsihw {
4527     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4528
4529     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4530
4531     my $scsihwid="$controller_prefix$controller";
4532     my $devices_list = vm_devices_list($vmid);
4533
4534     if(!defined($devices_list->{$scsihwid})) {
4535         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4536     }
4537
4538     return 1;
4539 }
4540
4541 sub qemu_deletescsihw {
4542     my ($conf, $vmid, $opt) = @_;
4543
4544     my $device = parse_drive($opt, $conf->{$opt});
4545
4546     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4547         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4548         return 1;
4549     }
4550
4551     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4552
4553     my $devices_list = vm_devices_list($vmid);
4554     foreach my $opt (keys %{$devices_list}) {
4555         if (PVE::QemuServer::is_valid_drivename($opt)) {
4556             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4557             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4558                 return 1;
4559             }
4560         }
4561     }
4562
4563     my $scsihwid="scsihw$controller";
4564
4565     vm_deviceunplug($vmid, $conf, $scsihwid);
4566
4567     return 1;
4568 }
4569
4570 sub qemu_add_pci_bridge {
4571     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4572
4573     my $bridges = {};
4574
4575     my $bridgeid;
4576
4577     print_pci_addr($device, $bridges, $arch, $machine_type);
4578
4579     while (my ($k, $v) = each %$bridges) {
4580         $bridgeid = $k;
4581     }
4582     return 1 if !defined($bridgeid) || $bridgeid < 1;
4583
4584     my $bridge = "pci.$bridgeid";
4585     my $devices_list = vm_devices_list($vmid);
4586
4587     if (!defined($devices_list->{$bridge})) {
4588         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4589     }
4590
4591     return 1;
4592 }
4593
4594 sub qemu_set_link_status {
4595     my ($vmid, $device, $up) = @_;
4596
4597     mon_cmd($vmid, "set_link", name => $device,
4598                up => $up ? JSON::true : JSON::false);
4599 }
4600
4601 sub qemu_netdevadd {
4602     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4603
4604     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4605     my %options =  split(/[=,]/, $netdev);
4606
4607     mon_cmd($vmid, "netdev_add",  %options);
4608     return 1;
4609 }
4610
4611 sub qemu_netdevdel {
4612     my ($vmid, $deviceid) = @_;
4613
4614     mon_cmd($vmid, "netdev_del", id => $deviceid);
4615 }
4616
4617 sub qemu_usb_hotplug {
4618     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4619
4620     return if !$device;
4621
4622     # remove the old one first
4623     vm_deviceunplug($vmid, $conf, $deviceid);
4624
4625     # check if xhci controller is necessary and available
4626     if ($device->{usb3}) {
4627
4628         my $devicelist = vm_devices_list($vmid);
4629
4630         if (!$devicelist->{xhci}) {
4631             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4632             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4633         }
4634     }
4635     my $d = parse_usb_device($device->{host});
4636     $d->{usb3} = $device->{usb3};
4637
4638     # add the new one
4639     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4640 }
4641
4642 sub qemu_cpu_hotplug {
4643     my ($vmid, $conf, $vcpus) = @_;
4644
4645     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4646
4647     my $sockets = 1;
4648     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4649     $sockets = $conf->{sockets} if  $conf->{sockets};
4650     my $cores = $conf->{cores} || 1;
4651     my $maxcpus = $sockets * $cores;
4652
4653     $vcpus = $maxcpus if !$vcpus;
4654
4655     die "you can't add more vcpus than maxcpus\n"
4656         if $vcpus > $maxcpus;
4657
4658     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4659
4660     if ($vcpus < $currentvcpus) {
4661
4662         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4663
4664             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4665                 qemu_devicedel($vmid, "cpu$i");
4666                 my $retry = 0;
4667                 my $currentrunningvcpus = undef;
4668                 while (1) {
4669                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus");
4670                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4671                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4672                     $retry++;
4673                     sleep 1;
4674                 }
4675                 #update conf after each succesfull cpu unplug
4676                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4677                 PVE::QemuConfig->write_config($vmid, $conf);
4678             }
4679         } else {
4680             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4681         }
4682
4683         return;
4684     }
4685
4686     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus");
4687     die "vcpus in running vm does not match its configuration\n"
4688         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4689
4690     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4691
4692         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4693             my $cpustr = print_cpu_device($conf, $i);
4694             qemu_deviceadd($vmid, $cpustr);
4695
4696             my $retry = 0;
4697             my $currentrunningvcpus = undef;
4698             while (1) {
4699                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus");
4700                 last if scalar(@{$currentrunningvcpus}) == $i;
4701                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4702                 sleep 1;
4703                 $retry++;
4704             }
4705             #update conf after each succesfull cpu hotplug
4706             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4707             PVE::QemuConfig->write_config($vmid, $conf);
4708         }
4709     } else {
4710
4711         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4712             mon_cmd($vmid, "cpu-add", id => int($i));
4713         }
4714     }
4715 }
4716
4717 sub qemu_block_set_io_throttle {
4718     my ($vmid, $deviceid,
4719         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4720         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4721         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4722         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4723
4724     return if !check_running($vmid) ;
4725
4726     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4727         bps => int($bps),
4728         bps_rd => int($bps_rd),
4729         bps_wr => int($bps_wr),
4730         iops => int($iops),
4731         iops_rd => int($iops_rd),
4732         iops_wr => int($iops_wr),
4733         bps_max => int($bps_max),
4734         bps_rd_max => int($bps_rd_max),
4735         bps_wr_max => int($bps_wr_max),
4736         iops_max => int($iops_max),
4737         iops_rd_max => int($iops_rd_max),
4738         iops_wr_max => int($iops_wr_max),
4739         bps_max_length => int($bps_max_length),
4740         bps_rd_max_length => int($bps_rd_max_length),
4741         bps_wr_max_length => int($bps_wr_max_length),
4742         iops_max_length => int($iops_max_length),
4743         iops_rd_max_length => int($iops_rd_max_length),
4744         iops_wr_max_length => int($iops_wr_max_length),
4745     );
4746
4747 }
4748
4749 # old code, only used to shutdown old VM after update
4750 sub __read_avail {
4751     my ($fh, $timeout) = @_;
4752
4753     my $sel = new IO::Select;
4754     $sel->add($fh);
4755
4756     my $res = '';
4757     my $buf;
4758
4759     my @ready;
4760     while (scalar (@ready = $sel->can_read($timeout))) {
4761         my $count;
4762         if ($count = $fh->sysread($buf, 8192)) {
4763             if ($buf =~ /^(.*)\(qemu\) $/s) {
4764                 $res .= $1;
4765                 last;
4766             } else {
4767                 $res .= $buf;
4768             }
4769         } else {
4770             if (!defined($count)) {
4771                 die "$!\n";
4772             }
4773             last;
4774         }
4775     }
4776
4777     die "monitor read timeout\n" if !scalar(@ready);
4778
4779     return $res;
4780 }
4781
4782 sub qemu_block_resize {
4783     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4784
4785     my $running = check_running($vmid);
4786
4787     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4788
4789     return if !$running;
4790
4791     mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4792
4793 }
4794
4795 sub qemu_volume_snapshot {
4796     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4797
4798     my $running = check_running($vmid);
4799
4800     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4801         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4802     } else {
4803         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4804     }
4805 }
4806
4807 sub qemu_volume_snapshot_delete {
4808     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4809
4810     my $running = check_running($vmid);
4811
4812     if($running) {
4813
4814         $running = undef;
4815         my $conf = PVE::QemuConfig->load_config($vmid);
4816         foreach_drive($conf, sub {
4817             my ($ds, $drive) = @_;
4818             $running = 1 if $drive->{file} eq $volid;
4819         });
4820     }
4821
4822     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4823         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4824     } else {
4825         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4826     }
4827 }
4828
4829 sub set_migration_caps {
4830     my ($vmid) = @_;
4831
4832     my $cap_ref = [];
4833
4834     my $enabled_cap = {
4835         "auto-converge" => 1,
4836         "xbzrle" => 1,
4837         "x-rdma-pin-all" => 0,
4838         "zero-blocks" => 0,
4839         "compress" => 0
4840     };
4841
4842     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4843
4844     for my $supported_capability (@$supported_capabilities) {
4845         push @$cap_ref, {
4846             capability => $supported_capability->{capability},
4847             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4848         };
4849     }
4850
4851     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4852 }
4853
4854 my $fast_plug_option = {
4855     'lock' => 1,
4856     'name' => 1,
4857     'onboot' => 1,
4858     'shares' => 1,
4859     'startup' => 1,
4860     'description' => 1,
4861     'protection' => 1,
4862     'vmstatestorage' => 1,
4863     'hookscript' => 1,
4864     'tags' => 1,
4865 };
4866
4867 # hotplug changes in [PENDING]
4868 # $selection hash can be used to only apply specified options, for
4869 # example: { cores => 1 } (only apply changed 'cores')
4870 # $errors ref is used to return error messages
4871 sub vmconfig_hotplug_pending {
4872     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4873
4874     my $defaults = load_defaults();
4875     my $arch = get_vm_arch($conf);
4876     my $machine_type = get_vm_machine($conf, undef, $arch);
4877
4878     # commit values which do not have any impact on running VM first
4879     # Note: those option cannot raise errors, we we do not care about
4880     # $selection and always apply them.
4881
4882     my $add_error = sub {
4883         my ($opt, $msg) = @_;
4884         $errors->{$opt} = "hotplug problem - $msg";
4885     };
4886
4887     my $changes = 0;
4888     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4889         if ($fast_plug_option->{$opt}) {
4890             $conf->{$opt} = $conf->{pending}->{$opt};
4891             delete $conf->{pending}->{$opt};
4892             $changes = 1;
4893         }
4894     }
4895
4896     if ($changes) {
4897         PVE::QemuConfig->write_config($vmid, $conf);
4898     }
4899
4900     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4901
4902     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4903     foreach my $opt (sort keys %$pending_delete_hash) {
4904         next if $selection && !$selection->{$opt};
4905         my $force = $pending_delete_hash->{$opt}->{force};
4906         eval {
4907             if ($opt eq 'hotplug') {
4908                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4909             } elsif ($opt eq 'tablet') {
4910                 die "skip\n" if !$hotplug_features->{usb};
4911                 if ($defaults->{tablet}) {
4912                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4913                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4914                         if $arch eq 'aarch64';
4915                 } else {
4916                     vm_deviceunplug($vmid, $conf, 'tablet');
4917                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4918                 }
4919             } elsif ($opt =~ m/^usb\d+/) {
4920                 die "skip\n";
4921                 # since we cannot reliably hot unplug usb devices
4922                 # we are disabling it
4923                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4924                 vm_deviceunplug($vmid, $conf, $opt);
4925             } elsif ($opt eq 'vcpus') {
4926                 die "skip\n" if !$hotplug_features->{cpu};
4927                 qemu_cpu_hotplug($vmid, $conf, undef);
4928             } elsif ($opt eq 'balloon') {
4929                 # enable balloon device is not hotpluggable
4930                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4931                 # here we reset the ballooning value to memory
4932                 my $balloon = $conf->{memory} || $defaults->{memory};
4933                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4934             } elsif ($fast_plug_option->{$opt}) {
4935                 # do nothing
4936             } elsif ($opt =~ m/^net(\d+)$/) {
4937                 die "skip\n" if !$hotplug_features->{network};
4938                 vm_deviceunplug($vmid, $conf, $opt);
4939             } elsif (is_valid_drivename($opt)) {
4940                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4941                 vm_deviceunplug($vmid, $conf, $opt);
4942                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4943             } elsif ($opt =~ m/^memory$/) {
4944                 die "skip\n" if !$hotplug_features->{memory};
4945                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4946             } elsif ($opt eq 'cpuunits') {
4947                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4948             } elsif ($opt eq 'cpulimit') {
4949                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4950             } else {
4951                 die "skip\n";
4952             }
4953         };
4954         if (my $err = $@) {
4955             &$add_error($opt, $err) if $err ne "skip\n";
4956         } else {
4957             delete $conf->{$opt};
4958             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4959         }
4960     }
4961
4962     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4963     $apply_pending_cloudinit = sub {
4964         return if $apply_pending_cloudinit_done; # once is enough
4965         $apply_pending_cloudinit_done = 1; # once is enough
4966
4967         my ($key, $value) = @_;
4968
4969         my @cloudinit_opts = keys %$confdesc_cloudinit;
4970         foreach my $opt (keys %{$conf->{pending}}) {
4971             next if !grep { $_ eq $opt } @cloudinit_opts;
4972             $conf->{$opt} = delete $conf->{pending}->{$opt};
4973         }
4974
4975         my $new_conf = { %$conf };
4976         $new_conf->{$key} = $value;
4977         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4978     };
4979
4980     foreach my $opt (keys %{$conf->{pending}}) {
4981         next if $selection && !$selection->{$opt};
4982         my $value = $conf->{pending}->{$opt};
4983         eval {
4984             if ($opt eq 'hotplug') {
4985                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4986             } elsif ($opt eq 'tablet') {
4987                 die "skip\n" if !$hotplug_features->{usb};
4988                 if ($value == 1) {
4989                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4990                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4991                         if $arch eq 'aarch64';
4992                 } elsif ($value == 0) {
4993                     vm_deviceunplug($vmid, $conf, 'tablet');
4994                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4995                 }
4996             } elsif ($opt =~ m/^usb\d+$/) {
4997                 die "skip\n";
4998                 # since we cannot reliably hot unplug usb devices
4999                 # we are disabling it
5000                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
5001                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
5002                 die "skip\n" if !$d;
5003                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
5004             } elsif ($opt eq 'vcpus') {
5005                 die "skip\n" if !$hotplug_features->{cpu};
5006                 qemu_cpu_hotplug($vmid, $conf, $value);
5007             } elsif ($opt eq 'balloon') {
5008                 # enable/disable balloning device is not hotpluggable
5009                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
5010                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
5011                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
5012
5013                 # allow manual ballooning if shares is set to zero
5014                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
5015                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
5016                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5017                 }
5018             } elsif ($opt =~ m/^net(\d+)$/) {
5019                 # some changes can be done without hotplug
5020                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
5021                                     $vmid, $opt, $value, $arch, $machine_type);
5022             } elsif (is_valid_drivename($opt)) {
5023                 die "skip\n" if $opt eq 'efidisk0';
5024                 # some changes can be done without hotplug
5025                 my $drive = parse_drive($opt, $value);
5026                 if (drive_is_cloudinit($drive)) {
5027                     &$apply_pending_cloudinit($opt, $value);
5028                 }
5029                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5030                                      $vmid, $opt, $value, 1, $arch, $machine_type);
5031             } elsif ($opt =~ m/^memory$/) { #dimms
5032                 die "skip\n" if !$hotplug_features->{memory};
5033                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
5034             } elsif ($opt eq 'cpuunits') {
5035                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
5036             } elsif ($opt eq 'cpulimit') {
5037                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
5038                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
5039             } else {
5040                 die "skip\n";  # skip non-hot-pluggable options
5041             }
5042         };
5043         if (my $err = $@) {
5044             &$add_error($opt, $err) if $err ne "skip\n";
5045         } else {
5046             $conf->{$opt} = $value;
5047             delete $conf->{pending}->{$opt};
5048         }
5049     }
5050
5051     PVE::QemuConfig->write_config($vmid, $conf);
5052 }
5053
5054 sub try_deallocate_drive {
5055     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5056
5057     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5058         my $volid = $drive->{file};
5059         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5060             my $sid = PVE::Storage::parse_volume_id($volid);
5061             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5062
5063             # check if the disk is really unused
5064             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5065                 if is_volume_in_use($storecfg, $conf, $key, $volid);
5066             PVE::Storage::vdisk_free($storecfg, $volid);
5067             return 1;
5068         } else {
5069             # If vm is not owner of this disk remove from config
5070             return 1;
5071         }
5072     }
5073
5074     return undef;
5075 }
5076
5077 sub vmconfig_delete_or_detach_drive {
5078     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5079
5080     my $drive = parse_drive($opt, $conf->{$opt});
5081
5082     my $rpcenv = PVE::RPCEnvironment::get();
5083     my $authuser = $rpcenv->get_user();
5084
5085     if ($force) {
5086         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5087         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5088     } else {
5089         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5090     }
5091 }
5092
5093
5094
5095 sub vmconfig_apply_pending {
5096     my ($vmid, $conf, $storecfg, $errors) = @_;
5097
5098     my $add_apply_error = sub {
5099         my ($opt, $msg) = @_;
5100         my $err_msg = "unable to apply pending change $opt : $msg";
5101         $errors->{$opt} = $err_msg;
5102         warn $err_msg;
5103     };
5104
5105     # cold plug
5106
5107     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5108     foreach my $opt (sort keys %$pending_delete_hash) {
5109         my $force = $pending_delete_hash->{$opt}->{force};
5110         eval {
5111             die "internal error" if $opt =~ m/^unused/;
5112             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5113             if (!defined($conf->{$opt})) {
5114                 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5115                 PVE::QemuConfig->write_config($vmid, $conf);
5116             } elsif (is_valid_drivename($opt)) {
5117                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5118                 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5119                 delete $conf->{$opt};
5120                 PVE::QemuConfig->write_config($vmid, $conf);
5121             } else {
5122                 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5123                 delete $conf->{$opt};
5124                 PVE::QemuConfig->write_config($vmid, $conf);
5125             }
5126         };
5127         if (my $err = $@) {
5128             $add_apply_error->($opt, $err);
5129         } else {
5130             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5131             delete $conf->{$opt};
5132             PVE::QemuConfig->write_config($vmid, $conf);
5133         }
5134     }
5135
5136     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5137
5138     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5139         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5140
5141         eval {
5142             if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
5143                 # skip if nothing changed
5144             } elsif (is_valid_drivename($opt)) {
5145                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5146                     if defined($conf->{$opt});
5147                 $conf->{$opt} = $conf->{pending}->{$opt};
5148             } else {
5149                 $conf->{$opt} = $conf->{pending}->{$opt};
5150             }
5151         };
5152         if (my $err = $@) {
5153             $add_apply_error->($opt, $err);
5154         } else {
5155             $conf->{$opt} = delete $conf->{pending}->{$opt};
5156             PVE::QemuConfig->cleanup_pending($conf);
5157         }
5158
5159         PVE::QemuConfig->write_config($vmid, $conf);
5160     }
5161 }
5162
5163 my $safe_num_ne = sub {
5164     my ($a, $b) = @_;
5165
5166     return 0 if !defined($a) && !defined($b);
5167     return 1 if !defined($a);
5168     return 1 if !defined($b);
5169
5170     return $a != $b;
5171 };
5172
5173 my $safe_string_ne = sub {
5174     my ($a, $b) = @_;
5175
5176     return 0 if !defined($a) && !defined($b);
5177     return 1 if !defined($a);
5178     return 1 if !defined($b);
5179
5180     return $a ne $b;
5181 };
5182
5183 sub vmconfig_update_net {
5184     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5185
5186     my $newnet = parse_net($value);
5187
5188     if ($conf->{$opt}) {
5189         my $oldnet = parse_net($conf->{$opt});
5190
5191         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5192             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5193             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5194             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5195
5196             # for non online change, we try to hot-unplug
5197             die "skip\n" if !$hotplug;
5198             vm_deviceunplug($vmid, $conf, $opt);
5199         } else {
5200
5201             die "internal error" if $opt !~ m/net(\d+)/;
5202             my $iface = "tap${vmid}i$1";
5203
5204             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5205                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5206                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5207                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5208                 PVE::Network::tap_unplug($iface);
5209                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5210             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5211                 # Rate can be applied on its own but any change above needs to
5212                 # include the rate in tap_plug since OVS resets everything.
5213                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5214             }
5215
5216             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5217                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5218             }
5219
5220             return 1;
5221         }
5222     }
5223
5224     if ($hotplug) {
5225         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5226     } else {
5227         die "skip\n";
5228     }
5229 }
5230
5231 sub vmconfig_update_disk {
5232     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
5233
5234     # fixme: do we need force?
5235
5236     my $drive = parse_drive($opt, $value);
5237
5238     if ($conf->{$opt}) {
5239
5240         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
5241
5242             my $media = $drive->{media} || 'disk';
5243             my $oldmedia = $old_drive->{media} || 'disk';
5244             die "unable to change media type\n" if $media ne $oldmedia;
5245
5246             if (!drive_is_cdrom($old_drive)) {
5247
5248                 if ($drive->{file} ne $old_drive->{file}) {
5249
5250                     die "skip\n" if !$hotplug;
5251
5252                     # unplug and register as unused
5253                     vm_deviceunplug($vmid, $conf, $opt);
5254                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5255
5256                 } else {
5257                     # update existing disk
5258
5259                     # skip non hotpluggable value
5260                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5261                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5262                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5263                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
5264                         die "skip\n";
5265                     }
5266
5267                     # apply throttle
5268                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5269                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5270                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5271                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5272                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5273                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5274                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5275                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5276                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5277                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5278                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5279                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5280                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5281                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5282                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5283                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5284                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5285                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5286
5287                         qemu_block_set_io_throttle($vmid,"drive-$opt",
5288                                                    ($drive->{mbps} || 0)*1024*1024,
5289                                                    ($drive->{mbps_rd} || 0)*1024*1024,
5290                                                    ($drive->{mbps_wr} || 0)*1024*1024,
5291                                                    $drive->{iops} || 0,
5292                                                    $drive->{iops_rd} || 0,
5293                                                    $drive->{iops_wr} || 0,
5294                                                    ($drive->{mbps_max} || 0)*1024*1024,
5295                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
5296                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
5297                                                    $drive->{iops_max} || 0,
5298                                                    $drive->{iops_rd_max} || 0,
5299                                                    $drive->{iops_wr_max} || 0,
5300                                                    $drive->{bps_max_length} || 1,
5301                                                    $drive->{bps_rd_max_length} || 1,
5302                                                    $drive->{bps_wr_max_length} || 1,
5303                                                    $drive->{iops_max_length} || 1,
5304                                                    $drive->{iops_rd_max_length} || 1,
5305                                                    $drive->{iops_wr_max_length} || 1);
5306
5307                     }
5308
5309                     return 1;
5310                 }
5311
5312             } else { # cdrom
5313
5314                 if ($drive->{file} eq 'none') {
5315                     mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5316                     if (drive_is_cloudinit($old_drive)) {
5317                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5318                     }
5319                 } else {
5320                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5321                     mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5322                     mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5323                 }
5324
5325                 return 1;
5326             }
5327         }
5328     }
5329
5330     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5331     # hotplug new disks
5332     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5333     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5334 }
5335
5336 sub vm_start {
5337     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5338         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5339
5340     PVE::QemuConfig->lock_config($vmid, sub {
5341         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5342
5343         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5344
5345         my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended');
5346
5347         PVE::QemuConfig->check_lock($conf)
5348             if !($skiplock || $is_suspended);
5349
5350         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5351
5352         # clean up leftover reboot request files
5353         eval { clear_reboot_request($vmid); };
5354         warn $@ if $@;
5355
5356         if (!$statefile && scalar(keys %{$conf->{pending}})) {
5357             vmconfig_apply_pending($vmid, $conf, $storecfg);
5358             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5359         }
5360
5361         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5362
5363         my $defaults = load_defaults();
5364
5365         # set environment variable useful inside network script
5366         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5367
5368         my $local_volumes = {};
5369
5370         if ($targetstorage) {
5371             foreach_drive($conf, sub {
5372                 my ($ds, $drive) = @_;
5373
5374                 return if drive_is_cdrom($drive);
5375
5376                 my $volid = $drive->{file};
5377
5378                 return if !$volid;
5379
5380                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5381
5382                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5383                 return if $scfg->{shared};
5384                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5385             });
5386
5387             my $format = undef;
5388
5389             foreach my $opt (sort keys %$local_volumes) {
5390
5391                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5392                 my $drive = parse_drive($opt, $conf->{$opt});
5393
5394                 #if remote storage is specified, use default format
5395                 if ($targetstorage && $targetstorage ne "1") {
5396                     $storeid = $targetstorage;
5397                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5398                     $format = $defFormat;
5399                 } else {
5400                     #else we use same format than original
5401                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5402                     $format = qemu_img_format($scfg, $volid);
5403                 }
5404
5405                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5406                 my $newdrive = $drive;
5407                 $newdrive->{format} = $format;
5408                 $newdrive->{file} = $newvolid;
5409                 my $drivestr = print_drive($newdrive);
5410                 $local_volumes->{$opt} = $drivestr;
5411                 #pass drive to conf for command line
5412                 $conf->{$opt} = $drivestr;
5413             }
5414         }
5415
5416         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5417
5418         if ($is_suspended) {
5419             # enforce machine type on suspended vm to ensure HW compatibility
5420             $forcemachine = $conf->{runningmachine};
5421             print "Resuming suspended VM\n";
5422         }
5423
5424         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5425
5426         my $migration_ip;
5427         my $get_migration_ip = sub {
5428             my ($cidr, $nodename) = @_;
5429
5430             return $migration_ip if defined($migration_ip);
5431
5432             if (!defined($cidr)) {
5433                 my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5434                 $cidr = $dc_conf->{migration}->{network};
5435             }
5436
5437             if (defined($cidr)) {
5438                 my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5439
5440                 die "could not get IP: no address configured on local " .
5441                     "node for network '$cidr'\n" if scalar(@$ips) == 0;
5442
5443                 die "could not get IP: multiple addresses configured on local " .
5444                     "node for network '$cidr'\n" if scalar(@$ips) > 1;
5445
5446                 $migration_ip = @$ips[0];
5447             }
5448
5449             $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5450                 if !defined($migration_ip);
5451
5452             return $migration_ip;
5453         };
5454
5455         my $migrate_uri;
5456         if ($statefile) {
5457             if ($statefile eq 'tcp') {
5458                 my $localip = "localhost";
5459                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5460                 my $nodename = nodename();
5461
5462                 if (!defined($migration_type)) {
5463                     if (defined($datacenterconf->{migration}->{type})) {
5464                         $migration_type = $datacenterconf->{migration}->{type};
5465                     } else {
5466                         $migration_type = 'secure';
5467                     }
5468                 }
5469
5470                 if ($migration_type eq 'insecure') {
5471                     $localip = $get_migration_ip->($migration_network, $nodename);
5472                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5473                 }
5474
5475                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5476                 my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5477                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5478                 push @$cmd, '-incoming', $migrate_uri;
5479                 push @$cmd, '-S';
5480
5481             } elsif ($statefile eq 'unix') {
5482                 # should be default for secure migrations as a ssh TCP forward
5483                 # tunnel is not deterministic reliable ready and fails regurarly
5484                 # to set up in time, so use UNIX socket forwards
5485                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5486                 unlink $socket_addr;
5487
5488                 $migrate_uri = "unix:$socket_addr";
5489
5490                 push @$cmd, '-incoming', $migrate_uri;
5491                 push @$cmd, '-S';
5492
5493             } elsif (-e $statefile) {
5494                 push @$cmd, '-loadstate', $statefile;
5495             } else {
5496                 my $statepath = PVE::Storage::path($storecfg, $statefile);
5497                 push @$vollist, $statefile;
5498                 push @$cmd, '-loadstate', $statepath;
5499             }
5500         } elsif ($paused) {
5501             push @$cmd, '-S';
5502         }
5503
5504         # host pci devices
5505         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5506           my $d = parse_hostpci($conf->{"hostpci$i"});
5507           next if !$d;
5508           my $pcidevices = $d->{pciid};
5509           foreach my $pcidevice (@$pcidevices) {
5510                 my $pciid = $pcidevice->{id};
5511
5512                 my $info = PVE::SysFSTools::pci_device_info("$pciid");
5513                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5514                 die "no pci device info for device '$pciid'\n" if !$info;
5515
5516                 if ($d->{mdev}) {
5517                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5518                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5519                 } else {
5520                     die "can't unbind/bind pci group to vfio '$pciid'\n"
5521                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5522                     die "can't reset pci device '$pciid'\n"
5523                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5524                 }
5525           }
5526         }
5527
5528         PVE::Storage::activate_volumes($storecfg, $vollist);
5529
5530         eval {
5531             run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5532                 outfunc => sub {}, errfunc => sub {});
5533         };
5534         # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5535         # timeout should be more than enough here...
5536         PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5537
5538         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5539                                                   : $defaults->{cpuunits};
5540
5541         my $start_timeout = ($conf->{hugepages} || $is_suspended) ? 300 : 30;
5542         my %run_params = (
5543             timeout => $statefile ? undef : $start_timeout,
5544             umask => 0077,
5545             noerr => 1,
5546         );
5547
5548         # when migrating, prefix QEMU output so other side can pick up any
5549         # errors that might occur and show the user
5550         if ($migratedfrom) {
5551             $run_params{quiet} = 1;
5552             $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5553         }
5554
5555         my %properties = (
5556             Slice => 'qemu.slice',
5557             KillMode => 'none',
5558             CPUShares => $cpuunits
5559         );
5560
5561         if (my $cpulimit = $conf->{cpulimit}) {
5562             $properties{CPUQuota} = int($cpulimit * 100);
5563         }
5564         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5565
5566         my $run_qemu = sub {
5567             PVE::Tools::run_fork sub {
5568                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5569
5570                 my $exitcode = run_command($cmd, %run_params);
5571                 die "QEMU exited with code $exitcode\n" if $exitcode;
5572             };
5573         };
5574
5575         if ($conf->{hugepages}) {
5576
5577             my $code = sub {
5578                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5579                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5580
5581                 PVE::QemuServer::Memory::hugepages_mount();
5582                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5583
5584                 eval { $run_qemu->() };
5585                 if (my $err = $@) {
5586                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5587                     die $err;
5588                 }
5589
5590                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5591             };
5592             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5593
5594         } else {
5595             eval { $run_qemu->() };
5596         }
5597
5598         if (my $err = $@) {
5599             # deactivate volumes if start fails
5600             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5601             die "start failed: $err";
5602         }
5603
5604         print "migration listens on $migrate_uri\n" if $migrate_uri;
5605
5606         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5607             eval { mon_cmd($vmid, "cont"); };
5608             warn $@ if $@;
5609         }
5610
5611         #start nbd server for storage migration
5612         if ($targetstorage) {
5613             my $nodename = nodename();
5614             my $localip = $get_migration_ip->($migration_network, $nodename);
5615             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5616             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5617
5618             mon_cmd($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${storage_migrate_port}" } } );
5619
5620             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5621
5622             foreach my $opt (sort keys %$local_volumes) {
5623                 my $volid = $local_volumes->{$opt};
5624                 mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5625                 my $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}:exportname=drive-$opt";
5626                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5627             }
5628         }
5629
5630         if ($migratedfrom) {
5631             eval {
5632                 set_migration_caps($vmid);
5633             };
5634             warn $@ if $@;
5635
5636             if ($spice_port) {
5637                 print "spice listens on port $spice_port\n";
5638                 if ($spice_ticket) {
5639                     mon_cmd($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5640                     mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5641                 }
5642             }
5643
5644         } else {
5645             mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5646                 if !$statefile && $conf->{balloon};
5647
5648             foreach my $opt (keys %$conf) {
5649                 next if $opt !~  m/^net\d+$/;
5650                 my $nicconf = parse_net($conf->{$opt});
5651                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5652             }
5653         }
5654
5655         mon_cmd($vmid, 'qom-set',
5656                     path => "machine/peripheral/balloon0",
5657                     property => "guest-stats-polling-interval",
5658                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5659
5660         if ($is_suspended) {
5661             print "Resumed VM, removing state\n";
5662             if (my $vmstate = $conf->{vmstate}) {
5663                 PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5664                 PVE::Storage::vdisk_free($storecfg, $vmstate);
5665             }
5666             delete $conf->@{qw(lock vmstate runningmachine)};
5667             PVE::QemuConfig->write_config($vmid, $conf);
5668         }
5669
5670         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5671     });
5672 }
5673
5674 sub vm_commandline {
5675     my ($storecfg, $vmid, $snapname) = @_;
5676
5677     my $conf = PVE::QemuConfig->load_config($vmid);
5678     my $forcemachine;
5679
5680     if ($snapname) {
5681         my $snapshot = $conf->{snapshots}->{$snapname};
5682         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5683
5684         # check for a 'runningmachine' in snapshot
5685         $forcemachine = $snapshot->{runningmachine} if $snapshot->{runningmachine};
5686
5687         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5688
5689         $conf = $snapshot;
5690     }
5691
5692     my $defaults = load_defaults();
5693
5694     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5695
5696     return PVE::Tools::cmd2string($cmd);
5697 }
5698
5699 sub vm_reset {
5700     my ($vmid, $skiplock) = @_;
5701
5702     PVE::QemuConfig->lock_config($vmid, sub {
5703
5704         my $conf = PVE::QemuConfig->load_config($vmid);
5705
5706         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5707
5708         mon_cmd($vmid, "system_reset");
5709     });
5710 }
5711
5712 sub get_vm_volumes {
5713     my ($conf) = @_;
5714
5715     my $vollist = [];
5716     foreach_volid($conf, sub {
5717         my ($volid, $attr) = @_;
5718
5719         return if $volid =~ m|^/|;
5720
5721         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5722         return if !$sid;
5723
5724         push @$vollist, $volid;
5725     });
5726
5727     return $vollist;
5728 }
5729
5730 sub vm_stop_cleanup {
5731     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5732
5733     eval {
5734
5735         if (!$keepActive) {
5736             my $vollist = get_vm_volumes($conf);
5737             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5738         }
5739
5740         foreach my $ext (qw(mon qmp pid vnc qga)) {
5741             unlink "/var/run/qemu-server/${vmid}.$ext";
5742         }
5743
5744         if ($conf->{ivshmem}) {
5745             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5746             # just delete it for now, VMs which have this already open do not
5747             # are affected, but new VMs will get a separated one. If this
5748             # becomes an issue we either add some sort of ref-counting or just
5749             # add a "don't delete on stop" flag to the ivshmem format.
5750             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5751         }
5752
5753         foreach my $key (keys %$conf) {
5754             next if $key !~ m/^hostpci(\d+)$/;
5755             my $hostpciindex = $1;
5756             my $d = parse_hostpci($conf->{$key});
5757             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5758
5759             foreach my $pci (@{$d->{pciid}}) {
5760                 my $pciid = $pci->{id};
5761                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5762             }
5763         }
5764
5765         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5766     };
5767     warn $@ if $@; # avoid errors - just warn
5768 }
5769
5770 # call only in locked context
5771 sub _do_vm_stop {
5772     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5773
5774     my $pid = check_running($vmid, $nocheck);
5775     return if !$pid;
5776
5777     my $conf;
5778     if (!$nocheck) {
5779         $conf = PVE::QemuConfig->load_config($vmid);
5780         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5781         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5782             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5783             $timeout = $opts->{down} if $opts->{down};
5784         }
5785         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5786     }
5787
5788     eval {
5789         if ($shutdown) {
5790             if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5791                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5792             } else {
5793                 mon_cmd($vmid, "system_powerdown");
5794             }
5795         } else {
5796             mon_cmd($vmid, "quit");
5797         }
5798     };
5799     my $err = $@;
5800
5801     if (!$err) {
5802         $timeout = 60 if !defined($timeout);
5803
5804         my $count = 0;
5805         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5806             $count++;
5807             sleep 1;
5808         }
5809
5810         if ($count >= $timeout) {
5811             if ($force) {
5812                 warn "VM still running - terminating now with SIGTERM\n";
5813                 kill 15, $pid;
5814             } else {
5815                 die "VM quit/powerdown failed - got timeout\n";
5816             }
5817         } else {
5818             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5819             return;
5820         }
5821     } else {
5822         if ($force) {
5823             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5824             kill 15, $pid;
5825         } else {
5826             die "VM quit/powerdown failed\n";
5827         }
5828     }
5829
5830     # wait again
5831     $timeout = 10;
5832
5833     my $count = 0;
5834     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5835         $count++;
5836         sleep 1;
5837     }
5838
5839     if ($count >= $timeout) {
5840         warn "VM still running - terminating now with SIGKILL\n";
5841         kill 9, $pid;
5842         sleep 1;
5843     }
5844
5845     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5846 }
5847
5848 # Note: use $nocheck to skip tests if VM configuration file exists.
5849 # We need that when migration VMs to other nodes (files already moved)
5850 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5851 sub vm_stop {
5852     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5853
5854     $force = 1 if !defined($force) && !$shutdown;
5855
5856     if ($migratedfrom){
5857         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5858         kill 15, $pid if $pid;
5859         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5860         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5861         return;
5862     }
5863
5864     PVE::QemuConfig->lock_config($vmid, sub {
5865         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5866    });
5867 }
5868
5869 sub vm_reboot {
5870     my ($vmid, $timeout) = @_;
5871
5872     PVE::QemuConfig->lock_config($vmid, sub {
5873         eval {
5874
5875             # only reboot if running, as qmeventd starts it again on a stop event
5876             return if !check_running($vmid);
5877
5878             create_reboot_request($vmid);
5879
5880             my $storecfg = PVE::Storage::config();
5881             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5882
5883         };
5884         if (my $err = $@) {
5885             # avoid that the next normal shutdown will be confused for a reboot
5886             clear_reboot_request($vmid);
5887             die $err;
5888         }
5889    });
5890 }
5891
5892 # note: if using the statestorage parameter, the caller has to check privileges
5893 sub vm_suspend {
5894     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5895
5896     my $conf;
5897     my $path;
5898     my $storecfg;
5899     my $vmstate;
5900
5901     PVE::QemuConfig->lock_config($vmid, sub {
5902
5903         $conf = PVE::QemuConfig->load_config($vmid);
5904
5905         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5906         PVE::QemuConfig->check_lock($conf)
5907             if !($skiplock || $is_backing_up);
5908
5909         die "cannot suspend to disk during backup\n"
5910             if $is_backing_up && $includestate;
5911
5912         if ($includestate) {
5913             $conf->{lock} = 'suspending';
5914             my $date = strftime("%Y-%m-%d", localtime(time()));
5915             $storecfg = PVE::Storage::config();
5916             if (!$statestorage) {
5917                 $statestorage = find_vmstate_storage($conf, $storecfg);
5918                 # check permissions for the storage
5919                 my $rpcenv = PVE::RPCEnvironment::get();
5920                 if ($rpcenv->{type} ne 'cli') {
5921                     my $authuser = $rpcenv->get_user();
5922                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5923                 }
5924             }
5925
5926
5927             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5928             $path = PVE::Storage::path($storecfg, $vmstate);
5929             PVE::QemuConfig->write_config($vmid, $conf);
5930         } else {
5931             mon_cmd($vmid, "stop");
5932         }
5933     });
5934
5935     if ($includestate) {
5936         # save vm state
5937         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5938
5939         eval {
5940             mon_cmd($vmid, "savevm-start", statefile => $path);
5941             for(;;) {
5942                 my $state = mon_cmd($vmid, "query-savevm");
5943                 if (!$state->{status}) {
5944                     die "savevm not active\n";
5945                 } elsif ($state->{status} eq 'active') {
5946                     sleep(1);
5947                     next;
5948                 } elsif ($state->{status} eq 'completed') {
5949                     print "State saved, quitting\n";
5950                     last;
5951                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5952                     die "query-savevm failed with error '$state->{error}'\n"
5953                 } else {
5954                     die "query-savevm returned status '$state->{status}'\n";
5955                 }
5956             }
5957         };
5958         my $err = $@;
5959
5960         PVE::QemuConfig->lock_config($vmid, sub {
5961             $conf = PVE::QemuConfig->load_config($vmid);
5962             if ($err) {
5963                 # cleanup, but leave suspending lock, to indicate something went wrong
5964                 eval {
5965                     mon_cmd($vmid, "savevm-end");
5966                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5967                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5968                     delete $conf->@{qw(vmstate runningmachine)};
5969                     PVE::QemuConfig->write_config($vmid, $conf);
5970                 };
5971                 warn $@ if $@;
5972                 die $err;
5973             }
5974
5975             die "lock changed unexpectedly\n"
5976                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5977
5978             mon_cmd($vmid, "quit");
5979             $conf->{lock} = 'suspended';
5980             PVE::QemuConfig->write_config($vmid, $conf);
5981         });
5982     }
5983 }
5984
5985 sub vm_resume {
5986     my ($vmid, $skiplock, $nocheck) = @_;
5987
5988     PVE::QemuConfig->lock_config($vmid, sub {
5989         my $res = mon_cmd($vmid, 'query-status');
5990         my $resume_cmd = 'cont';
5991
5992         if ($res->{status} && $res->{status} eq 'suspended') {
5993             $resume_cmd = 'system_wakeup';
5994         }
5995
5996         if (!$nocheck) {
5997
5998             my $conf = PVE::QemuConfig->load_config($vmid);
5999
6000             PVE::QemuConfig->check_lock($conf)
6001                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
6002         }
6003
6004         mon_cmd($vmid, $resume_cmd);
6005     });
6006 }
6007
6008 sub vm_sendkey {
6009     my ($vmid, $skiplock, $key) = @_;
6010
6011     PVE::QemuConfig->lock_config($vmid, sub {
6012
6013         my $conf = PVE::QemuConfig->load_config($vmid);
6014
6015         # there is no qmp command, so we use the human monitor command
6016         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
6017         die $res if $res ne '';
6018     });
6019 }
6020
6021 # vzdump restore implementaion
6022
6023 sub tar_archive_read_firstfile {
6024     my $archive = shift;
6025
6026     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6027
6028     # try to detect archive type first
6029     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6030         die "unable to open file '$archive'\n";
6031     my $firstfile = <$fh>;
6032     kill 15, $pid;
6033     close $fh;
6034
6035     die "ERROR: archive contaions no data\n" if !$firstfile;
6036     chomp $firstfile;
6037
6038     return $firstfile;
6039 }
6040
6041 sub tar_restore_cleanup {
6042     my ($storecfg, $statfile) = @_;
6043
6044     print STDERR "starting cleanup\n";
6045
6046     if (my $fd = IO::File->new($statfile, "r")) {
6047         while (defined(my $line = <$fd>)) {
6048             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6049                 my $volid = $2;
6050                 eval {
6051                     if ($volid =~ m|^/|) {
6052                         unlink $volid || die 'unlink failed\n';
6053                     } else {
6054                         PVE::Storage::vdisk_free($storecfg, $volid);
6055                     }
6056                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6057                 };
6058                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6059             } else {
6060                 print STDERR "unable to parse line in statfile - $line";
6061             }
6062         }
6063         $fd->close();
6064     }
6065 }
6066
6067 sub restore_archive {
6068     my ($archive, $vmid, $user, $opts) = @_;
6069
6070     my $format = $opts->{format};
6071     my $comp;
6072
6073     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
6074         $format = 'tar' if !$format;
6075         $comp = 'gzip';
6076     } elsif ($archive =~ m/\.tar$/) {
6077         $format = 'tar' if !$format;
6078     } elsif ($archive =~ m/.tar.lzo$/) {
6079         $format = 'tar' if !$format;
6080         $comp = 'lzop';
6081     } elsif ($archive =~ m/\.vma$/) {
6082         $format = 'vma' if !$format;
6083     } elsif ($archive =~ m/\.vma\.gz$/) {
6084         $format = 'vma' if !$format;
6085         $comp = 'gzip';
6086     } elsif ($archive =~ m/\.vma\.lzo$/) {
6087         $format = 'vma' if !$format;
6088         $comp = 'lzop';
6089     } else {
6090         $format = 'vma' if !$format; # default
6091     }
6092
6093     # try to detect archive format
6094     if ($format eq 'tar') {
6095         return restore_tar_archive($archive, $vmid, $user, $opts);
6096     } else {
6097         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6098     }
6099 }
6100
6101 sub restore_update_config_line {
6102     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
6103
6104     return if $line =~ m/^\#qmdump\#/;
6105     return if $line =~ m/^\#vzdump\#/;
6106     return if $line =~ m/^lock:/;
6107     return if $line =~ m/^unused\d+:/;
6108     return if $line =~ m/^parent:/;
6109
6110     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6111     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6112         # try to convert old 1.X settings
6113         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6114         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6115             my ($model, $macaddr) = split(/\=/, $devconfig);
6116             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6117             my $net = {
6118                 model => $model,
6119                 bridge => "vmbr$ind",
6120                 macaddr => $macaddr,
6121             };
6122             my $netstr = print_net($net);
6123
6124             print $outfd "net$cookie->{netcount}: $netstr\n";
6125             $cookie->{netcount}++;
6126         }
6127     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6128         my ($id, $netstr) = ($1, $2);
6129         my $net = parse_net($netstr);
6130         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6131         $netstr = print_net($net);
6132         print $outfd "$id: $netstr\n";
6133     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
6134         my $virtdev = $1;
6135         my $value = $3;
6136         my $di = parse_drive($virtdev, $value);
6137         if (defined($di->{backup}) && !$di->{backup}) {
6138             print $outfd "#$line";
6139         } elsif ($map->{$virtdev}) {
6140             delete $di->{format}; # format can change on restore
6141             $di->{file} = $map->{$virtdev};
6142             $value = print_drive($di);
6143             print $outfd "$virtdev: $value\n";
6144         } else {
6145             print $outfd $line;
6146         }
6147     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6148         my $vmgenid = $1;
6149         if ($vmgenid ne '0') {
6150             # always generate a new vmgenid if there was a valid one setup
6151             $vmgenid = generate_uuid();
6152         }
6153         print $outfd "vmgenid: $vmgenid\n";
6154     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6155         my ($uuid, $uuid_str);
6156         UUID::generate($uuid);
6157         UUID::unparse($uuid, $uuid_str);
6158         my $smbios1 = parse_smbios1($2);
6159         $smbios1->{uuid} = $uuid_str;
6160         print $outfd $1.print_smbios1($smbios1)."\n";
6161     } else {
6162         print $outfd $line;
6163     }
6164 }
6165
6166 sub scan_volids {
6167     my ($cfg, $vmid) = @_;
6168
6169     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
6170
6171     my $volid_hash = {};
6172     foreach my $storeid (keys %$info) {
6173         foreach my $item (@{$info->{$storeid}}) {
6174             next if !($item->{volid} && $item->{size});
6175             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6176             $volid_hash->{$item->{volid}} = $item;
6177         }
6178     }
6179
6180     return $volid_hash;
6181 }
6182
6183 sub is_volume_in_use {
6184     my ($storecfg, $conf, $skip_drive, $volid) = @_;
6185
6186     my $path = PVE::Storage::path($storecfg, $volid);
6187
6188     my $scan_config = sub {
6189         my ($cref, $snapname) = @_;
6190
6191         foreach my $key (keys %$cref) {
6192             my $value = $cref->{$key};
6193             if (is_valid_drivename($key)) {
6194                 next if $skip_drive && $key eq $skip_drive;
6195                 my $drive = parse_drive($key, $value);
6196                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
6197                 return 1 if $volid eq $drive->{file};
6198                 if ($drive->{file} =~ m!^/!) {
6199                     return 1 if $drive->{file} eq $path;
6200                 } else {
6201                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
6202                     next if !$storeid;
6203                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
6204                     next if !$scfg;
6205                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
6206                 }
6207             }
6208         }
6209
6210         return 0;
6211     };
6212
6213     return 1 if &$scan_config($conf);
6214
6215     undef $skip_drive;
6216
6217     foreach my $snapname (keys %{$conf->{snapshots}}) {
6218         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
6219     }
6220
6221     return 0;
6222 }
6223
6224 sub update_disksize {
6225     my ($drive, $volid_hash) = @_;
6226
6227     my $volid = $drive->{file};
6228     return undef if !defined($volid);
6229
6230     my $oldsize = $drive->{size};
6231     my $newsize = $volid_hash->{$volid}->{size};
6232
6233     if (defined($newsize) && defined($oldsize) && $newsize != $oldsize) {
6234         $drive->{size} = $newsize;
6235
6236         my $old_fmt = PVE::JSONSchema::format_size($oldsize);
6237         my $new_fmt = PVE::JSONSchema::format_size($newsize);
6238
6239         return wantarray ? ($drive, $old_fmt, $new_fmt) : $drive;
6240     }
6241
6242     return undef;
6243 }
6244
6245 sub update_disk_config {
6246     my ($vmid, $conf, $volid_hash) = @_;
6247
6248     my $changes;
6249     my $prefix = "VM $vmid:";
6250
6251     # used and unused disks
6252     my $referenced = {};
6253
6254     # Note: it is allowed to define multiple storages with same path (alias), so
6255     # we need to check both 'volid' and real 'path' (two different volid can point
6256     # to the same path).
6257
6258     my $referencedpath = {};
6259
6260     # update size info
6261     foreach my $opt (keys %$conf) {
6262         if (is_valid_drivename($opt)) {
6263             my $drive = parse_drive($opt, $conf->{$opt});
6264             my $volid = $drive->{file};
6265             next if !$volid;
6266
6267             # mark volid as "in-use" for next step
6268             $referenced->{$volid} = 1;
6269             if ($volid_hash->{$volid} &&
6270                 (my $path = $volid_hash->{$volid}->{path})) {
6271                 $referencedpath->{$path} = 1;
6272             }
6273
6274             next if drive_is_cdrom($drive);
6275             next if !$volid_hash->{$volid};
6276
6277             my ($updated, $old_size, $new_size) = update_disksize($drive, $volid_hash);
6278             if (defined($updated)) {
6279                 $changes = 1;
6280                 $conf->{$opt} = print_drive($updated);
6281                 print "$prefix size of disk '$volid' ($opt) updated from $old_size to $new_size\n";
6282             }
6283         }
6284     }
6285
6286     # remove 'unusedX' entry if volume is used
6287     foreach my $opt (keys %$conf) {
6288         next if $opt !~ m/^unused\d+$/;
6289         my $volid = $conf->{$opt};
6290         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6291         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6292             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6293             $changes = 1;
6294             delete $conf->{$opt};
6295         }
6296
6297         $referenced->{$volid} = 1;
6298         $referencedpath->{$path} = 1 if $path;
6299     }
6300
6301     foreach my $volid (sort keys %$volid_hash) {
6302         next if $volid =~ m/vm-$vmid-state-/;
6303         next if $referenced->{$volid};
6304         my $path = $volid_hash->{$volid}->{path};
6305         next if !$path; # just to be sure
6306         next if $referencedpath->{$path};
6307         $changes = 1;
6308         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6309         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6310         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6311     }
6312
6313     return $changes;
6314 }
6315
6316 sub rescan {
6317     my ($vmid, $nolock, $dryrun) = @_;
6318
6319     my $cfg = PVE::Storage::config();
6320
6321     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6322     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6323     foreach my $stor (keys %{$cfg->{ids}}) {
6324         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6325     }
6326
6327     print "rescan volumes...\n";
6328     my $volid_hash = scan_volids($cfg, $vmid);
6329
6330     my $updatefn =  sub {
6331         my ($vmid) = @_;
6332
6333         my $conf = PVE::QemuConfig->load_config($vmid);
6334
6335         PVE::QemuConfig->check_lock($conf);
6336
6337         my $vm_volids = {};
6338         foreach my $volid (keys %$volid_hash) {
6339             my $info = $volid_hash->{$volid};
6340             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6341         }
6342
6343         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6344
6345         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6346     };
6347
6348     if (defined($vmid)) {
6349         if ($nolock) {
6350             &$updatefn($vmid);
6351         } else {
6352             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6353         }
6354     } else {
6355         my $vmlist = config_list();
6356         foreach my $vmid (keys %$vmlist) {
6357             if ($nolock) {
6358                 &$updatefn($vmid);
6359             } else {
6360                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6361             }
6362         }
6363     }
6364 }
6365
6366 sub restore_vma_archive {
6367     my ($archive, $vmid, $user, $opts, $comp) = @_;
6368
6369     my $readfrom = $archive;
6370
6371     my $cfg = PVE::Storage::config();
6372     my $commands = [];
6373     my $bwlimit = $opts->{bwlimit};
6374
6375     my $dbg_cmdstring = '';
6376     my $add_pipe = sub {
6377         my ($cmd) = @_;
6378         push @$commands, $cmd;
6379         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6380         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6381         $readfrom = '-';
6382     };
6383
6384     my $input = undef;
6385     if ($archive eq '-') {
6386         $input = '<&STDIN';
6387     } else {
6388         # If we use a backup from a PVE defined storage we also consider that
6389         # storage's rate limit:
6390         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6391         if (defined($volid)) {
6392             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6393             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6394             if ($readlimit) {
6395                 print STDERR "applying read rate limit: $readlimit\n";
6396                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6397                 $add_pipe->($cstream);
6398             }
6399         }
6400     }
6401
6402     if ($comp) {
6403         my $cmd;
6404         if ($comp eq 'gzip') {
6405             $cmd = ['zcat', $readfrom];
6406         } elsif ($comp eq 'lzop') {
6407             $cmd = ['lzop', '-d', '-c', $readfrom];
6408         } else {
6409             die "unknown compression method '$comp'\n";
6410         }
6411         $add_pipe->($cmd);
6412     }
6413
6414     my $tmpdir = "/var/tmp/vzdumptmp$$";
6415     rmtree $tmpdir;
6416
6417     # disable interrupts (always do cleanups)
6418     local $SIG{INT} =
6419         local $SIG{TERM} =
6420         local $SIG{QUIT} =
6421         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6422
6423     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6424     POSIX::mkfifo($mapfifo, 0600);
6425     my $fifofh;
6426
6427     my $openfifo = sub {
6428         open($fifofh, '>', $mapfifo) || die $!;
6429     };
6430
6431     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6432
6433     my $oldtimeout;
6434     my $timeout = 5;
6435
6436     my $devinfo = {};
6437
6438     my $rpcenv = PVE::RPCEnvironment::get();
6439
6440     my $conffile = PVE::QemuConfig->config_file($vmid);
6441     my $tmpfn = "$conffile.$$.tmp";
6442
6443     # Note: $oldconf is undef if VM does not exist
6444     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6445     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6446
6447     my %storage_limits;
6448
6449     my $print_devmap = sub {
6450         my $virtdev_hash = {};
6451
6452         my $cfgfn = "$tmpdir/qemu-server.conf";
6453
6454         # we can read the config - that is already extracted
6455         my $fh = IO::File->new($cfgfn, "r") ||
6456             "unable to read qemu-server.conf - $!\n";
6457
6458         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6459         if (-f $fwcfgfn) {
6460             my $pve_firewall_dir = '/etc/pve/firewall';
6461             mkdir $pve_firewall_dir; # make sure the dir exists
6462             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6463         }
6464
6465         while (defined(my $line = <$fh>)) {
6466             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6467                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6468                 die "archive does not contain data for drive '$virtdev'\n"
6469                     if !$devinfo->{$devname};
6470                 if (defined($opts->{storage})) {
6471                     $storeid = $opts->{storage} || 'local';
6472                 } elsif (!$storeid) {
6473                     $storeid = 'local';
6474                 }
6475                 $format = 'raw' if !$format;
6476                 $devinfo->{$devname}->{devname} = $devname;
6477                 $devinfo->{$devname}->{virtdev} = $virtdev;
6478                 $devinfo->{$devname}->{format} = $format;
6479                 $devinfo->{$devname}->{storeid} = $storeid;
6480
6481                 # check permission on storage
6482                 my $pool = $opts->{pool}; # todo: do we need that?
6483                 if ($user ne 'root@pam') {
6484                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6485                 }
6486
6487                 $storage_limits{$storeid} = $bwlimit;
6488
6489                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6490             } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6491                 my $virtdev = $1;
6492                 my $drive = parse_drive($virtdev, $2);
6493                 if (drive_is_cloudinit($drive)) {
6494                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6495                     my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6496                     my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6497
6498                     my $d = {
6499                         format => $format,
6500                         storeid => $opts->{storage} // $storeid,
6501                         size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6502                         file => $drive->{file}, # to make drive_is_cloudinit check possible
6503                         name => "vm-$vmid-cloudinit",
6504                         is_cloudinit => 1,
6505                     };
6506                     $virtdev_hash->{$virtdev} = $d;
6507                 }
6508             }
6509         }
6510
6511         foreach my $key (keys %storage_limits) {
6512             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6513             next if !$limit;
6514             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6515             $storage_limits{$key} = $limit * 1024;
6516         }
6517
6518         foreach my $devname (keys %$devinfo) {
6519             die "found no device mapping information for device '$devname'\n"
6520                 if !$devinfo->{$devname}->{virtdev};
6521         }
6522
6523         # create empty/temp config
6524         if ($oldconf) {
6525             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6526             foreach_drive($oldconf, sub {
6527                 my ($ds, $drive) = @_;
6528
6529                 return if drive_is_cdrom($drive, 1);
6530
6531                 my $volid = $drive->{file};
6532                 return if !$volid || $volid =~ m|^/|;
6533
6534                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6535                 return if !$path || !$owner || ($owner != $vmid);
6536
6537                 # Note: only delete disk we want to restore
6538                 # other volumes will become unused
6539                 if ($virtdev_hash->{$ds}) {
6540                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6541                     if (my $err = $@) {
6542                         warn $err;
6543                     }
6544                 }
6545             });
6546
6547             # delete vmstate files, after the restore we have no snapshots anymore
6548             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6549                 my $snap = $oldconf->{snapshots}->{$snapname};
6550                 if ($snap->{vmstate}) {
6551                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6552                     if (my $err = $@) {
6553                         warn $err;
6554                     }
6555                 }
6556             }
6557         }
6558
6559         my $map = {};
6560         foreach my $virtdev (sort keys %$virtdev_hash) {
6561             my $d = $virtdev_hash->{$virtdev};
6562             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6563             my $storeid = $d->{storeid};
6564             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6565
6566             my $map_opts = '';
6567             if (my $limit = $storage_limits{$storeid}) {
6568                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6569             }
6570
6571             # test if requested format is supported
6572             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6573             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6574             $d->{format} = $defFormat if !$supported;
6575
6576             my $name;
6577             if ($d->{is_cloudinit}) {
6578                 $name = $d->{name};
6579                 $name .= ".$d->{format}" if $d->{format} ne 'raw';
6580             }
6581
6582             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6583             print STDERR "new volume ID is '$volid'\n";
6584             $d->{volid} = $volid;
6585
6586             PVE::Storage::activate_volumes($cfg, [$volid]);
6587
6588             my $write_zeros = 1;
6589             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6590                 $write_zeros = 0;
6591             }
6592
6593             if (!$d->{is_cloudinit}) {
6594                 my $path = PVE::Storage::path($cfg, $volid);
6595
6596                 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6597
6598                 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6599             }
6600             $map->{$virtdev} = $volid;
6601         }
6602
6603         $fh->seek(0, 0) || die "seek failed - $!\n";
6604
6605         my $outfd = new IO::File ($tmpfn, "w") ||
6606             die "unable to write config for VM $vmid\n";
6607
6608         my $cookie = { netcount => 0 };
6609         while (defined(my $line = <$fh>)) {
6610             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6611         }
6612
6613         $fh->close();
6614         $outfd->close();
6615     };
6616
6617     eval {
6618         # enable interrupts
6619         local $SIG{INT} =
6620             local $SIG{TERM} =
6621             local $SIG{QUIT} =
6622             local $SIG{HUP} =
6623             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6624         local $SIG{ALRM} = sub { die "got timeout\n"; };
6625
6626         $oldtimeout = alarm($timeout);
6627
6628         my $parser = sub {
6629             my $line = shift;
6630
6631             print "$line\n";
6632
6633             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6634                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6635                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6636             } elsif ($line =~ m/^CTIME: /) {
6637                 # we correctly received the vma config, so we can disable
6638                 # the timeout now for disk allocation (set to 10 minutes, so
6639                 # that we always timeout if something goes wrong)
6640                 alarm(600);
6641                 &$print_devmap();
6642                 print $fifofh "done\n";
6643                 my $tmp = $oldtimeout || 0;
6644                 $oldtimeout = undef;
6645                 alarm($tmp);
6646                 close($fifofh);
6647             }
6648         };
6649
6650         print "restore vma archive: $dbg_cmdstring\n";
6651         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6652     };
6653     my $err = $@;
6654
6655     alarm($oldtimeout) if $oldtimeout;
6656
6657     my $vollist = [];
6658     foreach my $devname (keys %$devinfo) {
6659         my $volid = $devinfo->{$devname}->{volid};
6660         push @$vollist, $volid if $volid;
6661     }
6662
6663     PVE::Storage::deactivate_volumes($cfg, $vollist);
6664
6665     unlink $mapfifo;
6666
6667     if ($err) {
6668         rmtree $tmpdir;
6669         unlink $tmpfn;
6670
6671         foreach my $devname (keys %$devinfo) {
6672             my $volid = $devinfo->{$devname}->{volid};
6673             next if !$volid;
6674             eval {
6675                 if ($volid =~ m|^/|) {
6676                     unlink $volid || die 'unlink failed\n';
6677                 } else {
6678                     PVE::Storage::vdisk_free($cfg, $volid);
6679                 }
6680                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6681             };
6682             print STDERR "unable to cleanup '$volid' - $@" if $@;
6683         }
6684         die $err;
6685     }
6686
6687     rmtree $tmpdir;
6688
6689     rename($tmpfn, $conffile) ||
6690         die "unable to commit configuration file '$conffile'\n";
6691
6692     PVE::Cluster::cfs_update(); # make sure we read new file
6693
6694     eval { rescan($vmid, 1); };
6695     warn $@ if $@;
6696 }
6697
6698 sub restore_tar_archive {
6699     my ($archive, $vmid, $user, $opts) = @_;
6700
6701     if ($archive ne '-') {
6702         my $firstfile = tar_archive_read_firstfile($archive);
6703         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6704             if $firstfile ne 'qemu-server.conf';
6705     }
6706
6707     my $storecfg = PVE::Storage::config();
6708
6709     # avoid zombie disks when restoring over an existing VM -> cleanup first
6710     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6711     # skiplock=1 because qmrestore has set the 'create' lock itself already
6712     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6713     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6714
6715     my $tocmd = "/usr/lib/qemu-server/qmextract";
6716
6717     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6718     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6719     $tocmd .= ' --prealloc' if $opts->{prealloc};
6720     $tocmd .= ' --info' if $opts->{info};
6721
6722     # tar option "xf" does not autodetect compression when read from STDIN,
6723     # so we pipe to zcat
6724     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6725         PVE::Tools::shellquote("--to-command=$tocmd");
6726
6727     my $tmpdir = "/var/tmp/vzdumptmp$$";
6728     mkpath $tmpdir;
6729
6730     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6731     local $ENV{VZDUMP_VMID} = $vmid;
6732     local $ENV{VZDUMP_USER} = $user;
6733
6734     my $conffile = PVE::QemuConfig->config_file($vmid);
6735     my $tmpfn = "$conffile.$$.tmp";
6736
6737     # disable interrupts (always do cleanups)
6738     local $SIG{INT} =
6739         local $SIG{TERM} =
6740         local $SIG{QUIT} =
6741         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6742
6743     eval {
6744         # enable interrupts
6745         local $SIG{INT} =
6746             local $SIG{TERM} =
6747             local $SIG{QUIT} =
6748             local $SIG{HUP} =
6749             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6750
6751         if ($archive eq '-') {
6752             print "extracting archive from STDIN\n";
6753             run_command($cmd, input => "<&STDIN");
6754         } else {
6755             print "extracting archive '$archive'\n";
6756             run_command($cmd);
6757         }
6758
6759         return if $opts->{info};
6760
6761         # read new mapping
6762         my $map = {};
6763         my $statfile = "$tmpdir/qmrestore.stat";
6764         if (my $fd = IO::File->new($statfile, "r")) {
6765             while (defined (my $line = <$fd>)) {
6766                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6767                     $map->{$1} = $2 if $1;
6768                 } else {
6769                     print STDERR "unable to parse line in statfile - $line\n";
6770                 }
6771             }
6772             $fd->close();
6773         }
6774
6775         my $confsrc = "$tmpdir/qemu-server.conf";
6776
6777         my $srcfd = new IO::File($confsrc, "r") ||
6778             die "unable to open file '$confsrc'\n";
6779
6780         my $outfd = new IO::File ($tmpfn, "w") ||
6781             die "unable to write config for VM $vmid\n";
6782
6783         my $cookie = { netcount => 0 };
6784         while (defined (my $line = <$srcfd>)) {
6785             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6786         }
6787
6788         $srcfd->close();
6789         $outfd->close();
6790     };
6791     if (my $err = $@) {
6792         unlink $tmpfn;
6793         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6794         die $err;
6795     }
6796
6797     rmtree $tmpdir;
6798
6799     rename $tmpfn, $conffile ||
6800         die "unable to commit configuration file '$conffile'\n";
6801
6802     PVE::Cluster::cfs_update(); # make sure we read new file
6803
6804     eval { rescan($vmid, 1); };
6805     warn $@ if $@;
6806 };
6807
6808 sub foreach_storage_used_by_vm {
6809     my ($conf, $func) = @_;
6810
6811     my $sidhash = {};
6812
6813     foreach_drive($conf, sub {
6814         my ($ds, $drive) = @_;
6815         return if drive_is_cdrom($drive);
6816
6817         my $volid = $drive->{file};
6818
6819         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6820         $sidhash->{$sid} = $sid if $sid;
6821     });
6822
6823     foreach my $sid (sort keys %$sidhash) {
6824         &$func($sid);
6825     }
6826 }
6827
6828 my $qemu_snap_storage = {
6829     rbd => 1,
6830 };
6831 sub do_snapshots_with_qemu {
6832     my ($storecfg, $volid) = @_;
6833
6834     my $storage_name = PVE::Storage::parse_volume_id($volid);
6835     my $scfg = $storecfg->{ids}->{$storage_name};
6836
6837     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6838         return 1;
6839     }
6840
6841     if ($volid =~ m/\.(qcow2|qed)$/){
6842         return 1;
6843     }
6844
6845     return undef;
6846 }
6847
6848 sub qga_check_running {
6849     my ($vmid, $nowarn) = @_;
6850
6851     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6852     if ($@) {
6853         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6854         return 0;
6855     }
6856     return 1;
6857 }
6858
6859 sub template_create {
6860     my ($vmid, $conf, $disk) = @_;
6861
6862     my $storecfg = PVE::Storage::config();
6863
6864     foreach_drive($conf, sub {
6865         my ($ds, $drive) = @_;
6866
6867         return if drive_is_cdrom($drive);
6868         return if $disk && $ds ne $disk;
6869
6870         my $volid = $drive->{file};
6871         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6872
6873         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6874         $drive->{file} = $voliddst;
6875         $conf->{$ds} = print_drive($drive);
6876         PVE::QemuConfig->write_config($vmid, $conf);
6877     });
6878 }
6879
6880 sub convert_iscsi_path {
6881     my ($path) = @_;
6882
6883     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6884         my $portal = $1;
6885         my $target = $2;
6886         my $lun = $3;
6887
6888         my $initiator_name = get_initiator_name();
6889
6890         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6891                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6892     }
6893
6894     die "cannot convert iscsi path '$path', unkown format\n";
6895 }
6896
6897 sub qemu_img_convert {
6898     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6899
6900     my $storecfg = PVE::Storage::config();
6901     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6902     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6903
6904     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6905
6906     my $cachemode;
6907     my $src_path;
6908     my $src_is_iscsi = 0;
6909     my $src_format;
6910
6911     if ($src_storeid) {
6912         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6913         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6914         $src_format = qemu_img_format($src_scfg, $src_volname);
6915         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6916         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6917         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6918     } elsif (-f $src_volid) {
6919         $src_path = $src_volid;
6920         if ($src_path =~ m/\.($QEMU_FORMAT_RE)$/) {
6921             $src_format = $1;
6922         }
6923     }
6924
6925     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6926
6927     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6928     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6929     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6930     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6931
6932     my $cmd = [];
6933     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6934     push @$cmd, '-l', "snapshot.name=$snapname"
6935         if $snapname && $src_format && $src_format eq "qcow2";
6936     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6937     push @$cmd, '-T', $cachemode if defined($cachemode);
6938
6939     if ($src_is_iscsi) {
6940         push @$cmd, '--image-opts';
6941         $src_path = convert_iscsi_path($src_path);
6942     } elsif ($src_format) {
6943         push @$cmd, '-f', $src_format;
6944     }
6945
6946     if ($dst_is_iscsi) {
6947         push @$cmd, '--target-image-opts';
6948         $dst_path = convert_iscsi_path($dst_path);
6949     } else {
6950         push @$cmd, '-O', $dst_format;
6951     }
6952
6953     push @$cmd, $src_path;
6954
6955     if (!$dst_is_iscsi && $is_zero_initialized) {
6956         push @$cmd, "zeroinit:$dst_path";
6957     } else {
6958         push @$cmd, $dst_path;
6959     }
6960
6961     my $parser = sub {
6962         my $line = shift;
6963         if($line =~ m/\((\S+)\/100\%\)/){
6964             my $percent = $1;
6965             my $transferred = int($size * $percent / 100);
6966             my $remaining = $size - $transferred;
6967
6968             print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6969         }
6970
6971     };
6972
6973     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6974     my $err = $@;
6975     die "copy failed: $err" if $err;
6976 }
6977
6978 sub qemu_img_format {
6979     my ($scfg, $volname) = @_;
6980
6981     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6982         return $1;
6983     } else {
6984         return "raw";
6985     }
6986 }
6987
6988 sub qemu_drive_mirror {
6989     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6990
6991     $jobs = {} if !$jobs;
6992
6993     my $qemu_target;
6994     my $format;
6995     $jobs->{"drive-$drive"} = {};
6996
6997     if ($dst_volid =~ /^nbd:/) {
6998         $qemu_target = $dst_volid;
6999         $format = "nbd";
7000     } else {
7001         my $storecfg = PVE::Storage::config();
7002         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7003
7004         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7005
7006         $format = qemu_img_format($dst_scfg, $dst_volname);
7007
7008         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7009
7010         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7011     }
7012
7013     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7014     $opts->{format} = $format if $format;
7015
7016     if (defined($bwlimit)) {
7017         $opts->{speed} = $bwlimit * 1024;
7018         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7019     } else {
7020         print "drive mirror is starting for drive-$drive\n";
7021     }
7022
7023     # if a job already runs for this device we get an error, catch it for cleanup
7024     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7025     if (my $err = $@) {
7026         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7027         warn "$@\n" if $@;
7028         die "mirroring error: $err\n";
7029     }
7030
7031     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
7032 }
7033
7034 sub qemu_drive_mirror_monitor {
7035     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
7036
7037     eval {
7038         my $err_complete = 0;
7039
7040         while (1) {
7041             die "storage migration timed out\n" if $err_complete > 300;
7042
7043             my $stats = mon_cmd($vmid, "query-block-jobs");
7044
7045             my $running_mirror_jobs = {};
7046             foreach my $stat (@$stats) {
7047                 next if $stat->{type} ne 'mirror';
7048                 $running_mirror_jobs->{$stat->{device}} = $stat;
7049             }
7050
7051             my $readycounter = 0;
7052
7053             foreach my $job (keys %$jobs) {
7054
7055                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
7056                     print "$job : finished\n";
7057                     delete $jobs->{$job};
7058                     next;
7059                 }
7060
7061                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
7062
7063                 my $busy = $running_mirror_jobs->{$job}->{busy};
7064                 my $ready = $running_mirror_jobs->{$job}->{ready};
7065                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
7066                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
7067                     my $remaining = $total - $transferred;
7068                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7069
7070                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
7071                 }
7072
7073                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
7074             }
7075
7076             last if scalar(keys %$jobs) == 0;
7077
7078             if ($readycounter == scalar(keys %$jobs)) {
7079                 print "all mirroring jobs are ready \n";
7080                 last if $skipcomplete; #do the complete later
7081
7082                 if ($vmiddst && $vmiddst != $vmid) {
7083                     my $agent_running = $qga && qga_check_running($vmid);
7084                     if ($agent_running) {
7085                         print "freeze filesystem\n";
7086                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7087                     } else {
7088                         print "suspend vm\n";
7089                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7090                     }
7091
7092                     # if we clone a disk for a new target vm, we don't switch the disk
7093                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7094
7095                     if ($agent_running) {
7096                         print "unfreeze filesystem\n";
7097                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7098                     } else {
7099                         print "resume vm\n";
7100                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
7101                     }
7102
7103                     last;
7104                 } else {
7105
7106                     foreach my $job (keys %$jobs) {
7107                         # try to switch the disk if source and destination are on the same guest
7108                         print "$job: Completing block job...\n";
7109
7110                         eval { mon_cmd($vmid, "block-job-complete", device => $job) };
7111                         if ($@ =~ m/cannot be completed/) {
7112                             print "$job: Block job cannot be completed, try again.\n";
7113                             $err_complete++;
7114                         }else {
7115                             print "$job: Completed successfully.\n";
7116                             $jobs->{$job}->{complete} = 1;
7117                         }
7118                     }
7119                 }
7120             }
7121             sleep 1;
7122         }
7123     };
7124     my $err = $@;
7125
7126     if ($err) {
7127         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7128         die "mirroring error: $err";
7129     }
7130
7131 }
7132
7133 sub qemu_blockjobs_cancel {
7134     my ($vmid, $jobs) = @_;
7135
7136     foreach my $job (keys %$jobs) {
7137         print "$job: Cancelling block job\n";
7138         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7139         $jobs->{$job}->{cancel} = 1;
7140     }
7141
7142     while (1) {
7143         my $stats = mon_cmd($vmid, "query-block-jobs");
7144
7145         my $running_jobs = {};
7146         foreach my $stat (@$stats) {
7147             $running_jobs->{$stat->{device}} = $stat;
7148         }
7149
7150         foreach my $job (keys %$jobs) {
7151
7152             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7153                 print "$job: Done.\n";
7154                 delete $jobs->{$job};
7155             }
7156         }
7157
7158         last if scalar(keys %$jobs) == 0;
7159
7160         sleep 1;
7161     }
7162 }
7163
7164 sub clone_disk {
7165     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7166         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
7167
7168     my $newvolid;
7169
7170     if (!$full) {
7171         print "create linked clone of drive $drivename ($drive->{file})\n";
7172         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7173         push @$newvollist, $newvolid;
7174     } else {
7175
7176         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7177         $storeid = $storage if $storage;
7178
7179         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7180         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
7181
7182         print "create full clone of drive $drivename ($drive->{file})\n";
7183         my $name = undef;
7184         if (drive_is_cloudinit($drive)) {
7185             $name = "vm-$newvmid-cloudinit";
7186             $name .= ".$dst_format" if $dst_format ne 'raw';
7187             $snapname = undef;
7188             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7189         }
7190         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
7191         push @$newvollist, $newvolid;
7192
7193         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7194
7195         if (drive_is_cloudinit($drive)) {
7196             goto no_data_clone;
7197         }
7198
7199         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7200         if (!$running || $snapname) {
7201             # TODO: handle bwlimits
7202             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7203         } else {
7204
7205             my $kvmver = get_running_qemu_version ($vmid);
7206             if (!min_version($kvmver, 2, 7)) {
7207                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7208                     if $drive->{iothread};
7209             }
7210
7211             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga, $bwlimit);
7212         }
7213     }
7214
7215 no_data_clone:
7216     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
7217
7218     my $disk = $drive;
7219     $disk->{format} = undef;
7220     $disk->{file} = $newvolid;
7221     $disk->{size} = $size;
7222
7223     return $disk;
7224 }
7225
7226 sub get_running_qemu_version {
7227     my ($vmid) = @_;
7228     my $res = mon_cmd($vmid, "query-version");
7229     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7230 }
7231
7232 sub qemu_use_old_bios_files {
7233     my ($machine_type) = @_;
7234
7235     return if !$machine_type;
7236
7237     my $use_old_bios_files = undef;
7238
7239     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7240         $machine_type = $1;
7241         $use_old_bios_files = 1;
7242     } else {
7243         my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
7244         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7245         # load new efi bios files on migration. So this hack is required to allow
7246         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7247         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7248         $use_old_bios_files = !min_version($version, 2, 4);
7249     }
7250
7251     return ($use_old_bios_files, $machine_type);
7252 }
7253
7254 sub create_efidisk($$$$$) {
7255     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7256
7257     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7258     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7259
7260     my $vars_size_b = -s $ovmf_vars;
7261     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7262     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7263     PVE::Storage::activate_volumes($storecfg, [$volid]);
7264
7265     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7266     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7267
7268     return ($volid, $size/1024);
7269 }
7270
7271 sub vm_iothreads_list {
7272     my ($vmid) = @_;
7273
7274     my $res = mon_cmd($vmid, 'query-iothreads');
7275
7276     my $iothreads = {};
7277     foreach my $iothread (@$res) {
7278         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7279     }
7280
7281     return $iothreads;
7282 }
7283
7284 sub scsihw_infos {
7285     my ($conf, $drive) = @_;
7286
7287     my $maxdev = 0;
7288
7289     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7290         $maxdev = 7;
7291     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7292         $maxdev = 1;
7293     } else {
7294         $maxdev = 256;
7295     }
7296
7297     my $controller = int($drive->{index} / $maxdev);
7298     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
7299
7300     return ($maxdev, $controller, $controller_prefix);
7301 }
7302
7303 sub add_hyperv_enlightenments {
7304     my ($cpuFlags, $winversion, $machine_version, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
7305
7306     return if $winversion < 6;
7307     return if $bios && $bios eq 'ovmf' && $winversion < 8;
7308
7309     if ($gpu_passthrough || defined($hv_vendor_id)) {
7310         $hv_vendor_id //= 'proxmox';
7311         push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
7312     }
7313
7314     if (min_version($machine_version, 2, 3)) {
7315         push @$cpuFlags , 'hv_spinlocks=0x1fff';
7316         push @$cpuFlags , 'hv_vapic';
7317         push @$cpuFlags , 'hv_time';
7318     } else {
7319         push @$cpuFlags , 'hv_spinlocks=0xffff';
7320     }
7321
7322     if (min_version($machine_version, 2, 6)) {
7323         push @$cpuFlags , 'hv_reset';
7324         push @$cpuFlags , 'hv_vpindex';
7325         push @$cpuFlags , 'hv_runtime';
7326     }
7327
7328     if ($winversion >= 7) {
7329         push @$cpuFlags , 'hv_relaxed';
7330
7331         if (min_version($machine_version, 2, 12)) {
7332             push @$cpuFlags , 'hv_synic';
7333             push @$cpuFlags , 'hv_stimer';
7334         }
7335
7336         if (min_version($machine_version, 3, 1)) {
7337             push @$cpuFlags , 'hv_ipi';
7338         }
7339     }
7340 }
7341
7342 sub windows_version {
7343     my ($ostype) = @_;
7344
7345     return 0 if !$ostype;
7346
7347     my $winversion = 0;
7348
7349     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7350         $winversion = 5;
7351     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7352         $winversion = 6;
7353     } elsif ($ostype =~ m/^win(\d+)$/) {
7354         $winversion = $1;
7355     }
7356
7357     return $winversion;
7358 }
7359
7360 sub resolve_dst_disk_format {
7361         my ($storecfg, $storeid, $src_volname, $format) = @_;
7362         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7363
7364         if (!$format) {
7365             # if no target format is specified, use the source disk format as hint
7366             if ($src_volname) {
7367                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7368                 $format = qemu_img_format($scfg, $src_volname);
7369             } else {
7370                 return $defFormat;
7371             }
7372         }
7373
7374         # test if requested format is supported - else use default
7375         my $supported = grep { $_ eq $format } @$validFormats;
7376         $format = $defFormat if !$supported;
7377         return $format;
7378 }
7379
7380 sub resolve_first_disk {
7381     my $conf = shift;
7382     my @disks = PVE::QemuServer::valid_drive_names();
7383     my $firstdisk;
7384     foreach my $ds (reverse @disks) {
7385         next if !$conf->{$ds};
7386         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
7387         next if PVE::QemuServer::drive_is_cdrom($disk);
7388         $firstdisk = $ds;
7389     }
7390     return $firstdisk;
7391 }
7392
7393 # NOTE: if this logic changes, please update docs & possibly gui logic
7394 sub find_vmstate_storage {
7395     my ($conf, $storecfg) = @_;
7396
7397     # first, return storage from conf if set
7398     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7399
7400     my ($target, $shared, $local);
7401
7402     foreach_storage_used_by_vm($conf, sub {
7403         my ($sid) = @_;
7404         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7405         my $dst = $scfg->{shared} ? \$shared : \$local;
7406         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7407     });
7408
7409     # second, use shared storage where VM has at least one disk
7410     # third, use local storage where VM has at least one disk
7411     # fall back to local storage
7412     $target = $shared // $local // 'local';
7413
7414     return $target;
7415 }
7416
7417 sub generate_uuid {
7418     my ($uuid, $uuid_str);
7419     UUID::generate($uuid);
7420     UUID::unparse($uuid, $uuid_str);
7421     return $uuid_str;
7422 }
7423
7424 sub generate_smbios1_uuid {
7425     return "uuid=".generate_uuid();
7426 }
7427
7428 sub nbd_stop {
7429     my ($vmid) = @_;
7430
7431     mon_cmd($vmid, 'nbd-server-stop');
7432 }
7433
7434 sub create_reboot_request {
7435     my ($vmid) = @_;
7436     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7437         or die "failed to create reboot trigger file: $!\n";
7438     close($fh);
7439 }
7440
7441 sub clear_reboot_request {
7442     my ($vmid) = @_;
7443     my $path = "/run/qemu-server/$vmid.reboot";
7444     my $res = 0;
7445
7446     $res = unlink($path);
7447     die "could not remove reboot request for $vmid: $!"
7448         if !$res && $! != POSIX::ENOENT;
7449
7450     return $res;
7451 }
7452
7453 # bash completion helper
7454
7455 sub complete_backup_archives {
7456     my ($cmdname, $pname, $cvalue) = @_;
7457
7458     my $cfg = PVE::Storage::config();
7459
7460     my $storeid;
7461
7462     if ($cvalue =~ m/^([^:]+):/) {
7463         $storeid = $1;
7464     }
7465
7466     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7467
7468     my $res = [];
7469     foreach my $id (keys %$data) {
7470         foreach my $item (@{$data->{$id}}) {
7471             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
7472             push @$res, $item->{volid} if defined($item->{volid});
7473         }
7474     }
7475
7476     return $res;
7477 }
7478
7479 my $complete_vmid_full = sub {
7480     my ($running) = @_;
7481
7482     my $idlist = vmstatus();
7483
7484     my $res = [];
7485
7486     foreach my $id (keys %$idlist) {
7487         my $d = $idlist->{$id};
7488         if (defined($running)) {
7489             next if $d->{template};
7490             next if $running && $d->{status} ne 'running';
7491             next if !$running && $d->{status} eq 'running';
7492         }
7493         push @$res, $id;
7494
7495     }
7496     return $res;
7497 };
7498
7499 sub complete_vmid {
7500     return &$complete_vmid_full();
7501 }
7502
7503 sub complete_vmid_stopped {
7504     return &$complete_vmid_full(0);
7505 }
7506
7507 sub complete_vmid_running {
7508     return &$complete_vmid_full(1);
7509 }
7510
7511 sub complete_storage {
7512
7513     my $cfg = PVE::Storage::config();
7514     my $ids = $cfg->{ids};
7515
7516     my $res = [];
7517     foreach my $sid (keys %$ids) {
7518         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7519         next if !$ids->{$sid}->{content}->{images};
7520         push @$res, $sid;
7521     }
7522
7523     return $res;
7524 }
7525
7526 sub complete_migration_storage {
7527     my ($cmd, $param, $current_value, $all_args) = @_;
7528
7529     my $targetnode = @$all_args[1];
7530
7531     my $cfg = PVE::Storage::config();
7532     my $ids = $cfg->{ids};
7533
7534     my $res = [];
7535     foreach my $sid (keys %$ids) {
7536         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7537         next if !$ids->{$sid}->{content}->{images};
7538         push @$res, $sid;
7539     }
7540
7541     return $res;
7542 }
7543
7544 1;