PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  30 use PVE::DataCenterConfig;
  31 use PVE::Exception qw(raise raise_param_exc);
  32 use PVE::GuestHelpers;
  33 use PVE::INotify;
  34 use PVE::JSONSchema qw(get_standard_option);
  35 use PVE::ProcFSTools;
  36 use PVE::RPCEnvironment;
  37 use PVE::Storage;
  38 use PVE::SysFSTools;
  39 use PVE::Systemd;
  40 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  41
  42 use PVE::QMPClient;
  43 use PVE::QemuConfig;
  44 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  45 use PVE::QemuServer::Cloudinit;
  46 use PVE::QemuServer::Machine;
  47 use PVE::QemuServer::Memory;
  48 use PVE::QemuServer::Monitor qw(mon_cmd);
  49 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port);
  50 use PVE::QemuServer::USB qw(parse_usb_device);
  51
  52 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  53 my $OVMF = {
  54     x86_64 => [
  55         "$EDK2_FW_BASE/OVMF_CODE.fd",
  56         "$EDK2_FW_BASE/OVMF_VARS.fd"
  57     ],
  58     aarch64 => [
  59         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  60         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  61     ],
  62 };
  63
  64 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  65
  66 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  67
  68 # Note about locking: we use flock on the config file protect
  69 # against concurent actions.
  70 # Aditionaly, we have a 'lock' setting in the config file. This
  71 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  72 # allowed when such lock is set. But you can ignore this kind of
  73 # lock with the --skiplock flag.
  74
  75 cfs_register_file('/qemu-server/',
  76                   \&parse_vm_config,
  77                   \&write_vm_config);
  78
  79 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  80     description => "Some command save/restore state from this location.",
  81     type => 'string',
  82     maxLength => 128,
  83     optional => 1,
  84 });
  85
  86 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  87     type => 'string',
  88     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  89     description => "The drive's backing file's data format.",
  90     optional => 1,
  91 });
  92
  93 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  94         description => "Specifies the Qemu machine type.",
  95         type => 'string',
  96         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
  97         maxLength => 40,
  98         optional => 1,
  99 });
 100
 101 #no warnings 'redefine';
 102
 103 sub cgroups_write {
 104    my ($controller, $vmid, $option, $value) = @_;
 105
 106    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 107    PVE::ProcFSTools::write_proc_entry($path, $value);
 108
 109 }
 110
 111 my $nodename_cache;
 112 sub nodename {
 113     $nodename_cache //= PVE::INotify::nodename();
 114     return $nodename_cache;
 115 }
 116
 117 my $cpu_vendor_list = {
 118     # Intel CPUs
 119     486 => 'GenuineIntel',
 120     pentium => 'GenuineIntel',
 121     pentium2  => 'GenuineIntel',
 122     pentium3  => 'GenuineIntel',
 123     coreduo => 'GenuineIntel',
 124     core2duo => 'GenuineIntel',
 125     Conroe  => 'GenuineIntel',
 126     Penryn  => 'GenuineIntel',
 127     Nehalem  => 'GenuineIntel',
 128     'Nehalem-IBRS'  => 'GenuineIntel',
 129     Westmere => 'GenuineIntel',
 130     'Westmere-IBRS' => 'GenuineIntel',
 131     SandyBridge => 'GenuineIntel',
 132     'SandyBridge-IBRS' => 'GenuineIntel',
 133     IvyBridge => 'GenuineIntel',
 134     'IvyBridge-IBRS' => 'GenuineIntel',
 135     Haswell => 'GenuineIntel',
 136     'Haswell-IBRS' => 'GenuineIntel',
 137     'Haswell-noTSX' => 'GenuineIntel',
 138     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 139     Broadwell => 'GenuineIntel',
 140     'Broadwell-IBRS' => 'GenuineIntel',
 141     'Broadwell-noTSX' => 'GenuineIntel',
 142     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 143     'Skylake-Client' => 'GenuineIntel',
 144     'Skylake-Client-IBRS' => 'GenuineIntel',
 145     'Skylake-Server' => 'GenuineIntel',
 146     'Skylake-Server-IBRS' => 'GenuineIntel',
 147     'Cascadelake-Server' => 'GenuineIntel',
 148     KnightsMill => 'GenuineIntel',
 149
 150
 151     # AMD CPUs
 152     athlon => 'AuthenticAMD',
 153     phenom  => 'AuthenticAMD',
 154     Opteron_G1  => 'AuthenticAMD',
 155     Opteron_G2  => 'AuthenticAMD',
 156     Opteron_G3  => 'AuthenticAMD',
 157     Opteron_G4  => 'AuthenticAMD',
 158     Opteron_G5  => 'AuthenticAMD',
 159     EPYC => 'AuthenticAMD',
 160     'EPYC-IBPB' => 'AuthenticAMD',
 161
 162     # generic types, use vendor from host node
 163     host => 'default',
 164     kvm32 => 'default',
 165     kvm64 => 'default',
 166     qemu32 => 'default',
 167     qemu64 => 'default',
 168     max => 'default',
 169 };
 170
 171 my @supported_cpu_flags = (
 172     'pcid',
 173     'spec-ctrl',
 174     'ibpb',
 175     'ssbd',
 176     'virt-ssbd',
 177     'amd-ssbd',
 178     'amd-no-ssb',
 179     'pdpe1gb',
 180     'md-clear',
 181     'hv-tlbflush',
 182     'hv-evmcs',
 183     'aes'
 184 );
 185 my $cpu_flag = qr/[+-](@{[join('|', @supported_cpu_flags)]})/;
 186
 187 my $cpu_fmt = {
 188     cputype => {
 189         description => "Emulated CPU type.",
 190         type => 'string',
 191         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 192         default => 'kvm64',
 193         default_key => 1,
 194     },
 195     hidden => {
 196         description => "Do not identify as a KVM virtual machine.",
 197         type => 'boolean',
 198         optional => 1,
 199         default => 0
 200     },
 201     'hv-vendor-id' => {
 202         type => 'string',
 203         pattern => qr/[a-zA-Z0-9]{1,12}/,
 204         format_description => 'vendor-id',
 205         description => 'The Hyper-V vendor ID. Some drivers or programs inside Windows guests need a specific ID.',
 206         optional => 1,
 207     },
 208     flags => {
 209         description => "List of additional CPU flags separated by ';'."
 210                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 211                      . " Currently supported flags: @{[join(', ', @supported_cpu_flags)]}.",
 212         format_description => '+FLAG[;-FLAG...]',
 213         type => 'string',
 214         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 215         optional => 1,
 216     },
 217 };
 218
 219 my $watchdog_fmt = {
 220     model => {
 221         default_key => 1,
 222         type => 'string',
 223         enum => [qw(i6300esb ib700)],
 224         description => "Watchdog type to emulate.",
 225         default => 'i6300esb',
 226         optional => 1,
 227     },
 228     action => {
 229         type => 'string',
 230         enum => [qw(reset shutdown poweroff pause debug none)],
 231         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 232         optional => 1,
 233     },
 234 };
 235 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 236
 237 my $agent_fmt = {
 238     enabled => {
 239         description => "Enable/disable Qemu GuestAgent.",
 240         type => 'boolean',
 241         default => 0,
 242         default_key => 1,
 243     },
 244     fstrim_cloned_disks => {
 245         description => "Run fstrim after cloning/moving a disk.",
 246         type => 'boolean',
 247         optional => 1,
 248         default => 0
 249     },
 250     type => {
 251         description => "Select the agent type",
 252         type => 'string',
 253         default => 'virtio',
 254         optional => 1,
 255         enum => [qw(virtio isa)],
 256     },
 257 };
 258
 259 my $vga_fmt = {
 260     type => {
 261         description => "Select the VGA type.",
 262         type => 'string',
 263         default => 'std',
 264         optional => 1,
 265         default_key => 1,
 266         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 267     },
 268     memory => {
 269         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 270         type => 'integer',
 271         optional => 1,
 272         minimum => 4,
 273         maximum => 512,
 274     },
 275 };
 276
 277 my $ivshmem_fmt = {
 278     size => {
 279         type => 'integer',
 280         minimum => 1,
 281         description => "The size of the file in MB.",
 282     },
 283     name => {
 284         type => 'string',
 285         pattern => '[a-zA-Z0-9\-]+',
 286         optional => 1,
 287         format_description => 'string',
 288         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 289     },
 290 };
 291
 292 my $audio_fmt = {
 293     device => {
 294         type => 'string',
 295         enum => [qw(ich9-intel-hda intel-hda AC97)],
 296         description =>  "Configure an audio device."
 297     },
 298     driver =>  {
 299         type => 'string',
 300         enum => ['spice'],
 301         default => 'spice',
 302         optional => 1,
 303         description => "Driver backend for the audio device."
 304     },
 305 };
 306
 307 my $spice_enhancements_fmt = {
 308     foldersharing => {
 309         type => 'boolean',
 310         optional => 1,
 311         default => '0',
 312         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 313     },
 314     videostreaming =>  {
 315         type => 'string',
 316         enum => ['off', 'all', 'filter'],
 317         default => 'off',
 318         optional => 1,
 319         description => "Enable video streaming. Uses compression for detected video streams."
 320     },
 321 };
 322
 323 my $confdesc = {
 324     onboot => {
 325         optional => 1,
 326         type => 'boolean',
 327         description => "Specifies whether a VM will be started during system bootup.",
 328         default => 0,
 329     },
 330     autostart => {
 331         optional => 1,
 332         type => 'boolean',
 333         description => "Automatic restart after crash (currently ignored).",
 334         default => 0,
 335     },
 336     hotplug => {
 337         optional => 1,
 338         type => 'string', format => 'pve-hotplug-features',
 339         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 340         default => 'network,disk,usb',
 341     },
 342     reboot => {
 343         optional => 1,
 344         type => 'boolean',
 345         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 346         default => 1,
 347     },
 348     lock => {
 349         optional => 1,
 350         type => 'string',
 351         description => "Lock/unlock the VM.",
 352         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 353     },
 354     cpulimit => {
 355         optional => 1,
 356         type => 'number',
 357         description => "Limit of CPU usage.",
 358         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 359         minimum => 0,
 360         maximum => 128,
 361         default => 0,
 362     },
 363     cpuunits => {
 364         optional => 1,
 365         type => 'integer',
 366         description => "CPU weight for a VM.",
 367         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 368         minimum => 2,
 369         maximum => 262144,
 370         default => 1024,
 371     },
 372     memory => {
 373         optional => 1,
 374         type => 'integer',
 375         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 376         minimum => 16,
 377         default => 512,
 378     },
 379     balloon => {
 380         optional => 1,
 381         type => 'integer',
 382         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 383         minimum => 0,
 384     },
 385     shares => {
 386         optional => 1,
 387         type => 'integer',
 388         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 389         minimum => 0,
 390         maximum => 50000,
 391         default => 1000,
 392     },
 393     keyboard => {
 394         optional => 1,
 395         type => 'string',
 396         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 397                        "It should not be necessary to set it.",
 398         enum => PVE::Tools::kvmkeymaplist(),
 399         default => undef,
 400     },
 401     name => {
 402         optional => 1,
 403         type => 'string', format => 'dns-name',
 404         description => "Set a name for the VM. Only used on the configuration web interface.",
 405     },
 406     scsihw => {
 407         optional => 1,
 408         type => 'string',
 409         description => "SCSI controller model",
 410         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 411         default => 'lsi',
 412     },
 413     description => {
 414         optional => 1,
 415         type => 'string',
 416         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 417     },
 418     ostype => {
 419         optional => 1,
 420         type => 'string',
 421         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 422         description => "Specify guest operating system.",
 423         verbose_description => <<EODESC,
 424 Specify guest operating system. This is used to enable special
 425 optimization/features for specific operating systems:
 426
 427 [horizontal]
 428 other;; unspecified OS
 429 wxp;; Microsoft Windows XP
 430 w2k;; Microsoft Windows 2000
 431 w2k3;; Microsoft Windows 2003
 432 w2k8;; Microsoft Windows 2008
 433 wvista;; Microsoft Windows Vista
 434 win7;; Microsoft Windows 7
 435 win8;; Microsoft Windows 8/2012/2012r2
 436 win10;; Microsoft Windows 10/2016
 437 l24;; Linux 2.4 Kernel
 438 l26;; Linux 2.6 - 5.X Kernel
 439 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 440 EODESC
 441     },
 442     boot => {
 443         optional => 1,
 444         type => 'string',
 445         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 446         pattern => '[acdn]{1,4}',
 447         default => 'cdn',
 448     },
 449     bootdisk => {
 450         optional => 1,
 451         type => 'string', format => 'pve-qm-bootdisk',
 452         description => "Enable booting from specified disk.",
 453         pattern => '(ide|sata|scsi|virtio)\d+',
 454     },
 455     smp => {
 456         optional => 1,
 457         type => 'integer',
 458         description => "The number of CPUs. Please use option -sockets instead.",
 459         minimum => 1,
 460         default => 1,
 461     },
 462     sockets => {
 463         optional => 1,
 464         type => 'integer',
 465         description => "The number of CPU sockets.",
 466         minimum => 1,
 467         default => 1,
 468     },
 469     cores => {
 470         optional => 1,
 471         type => 'integer',
 472         description => "The number of cores per socket.",
 473         minimum => 1,
 474         default => 1,
 475     },
 476     numa => {
 477         optional => 1,
 478         type => 'boolean',
 479         description => "Enable/disable NUMA.",
 480         default => 0,
 481     },
 482     hugepages => {
 483         optional => 1,
 484         type => 'string',
 485         description => "Enable/disable hugepages memory.",
 486         enum => [qw(any 2 1024)],
 487     },
 488     vcpus => {
 489         optional => 1,
 490         type => 'integer',
 491         description => "Number of hotplugged vcpus.",
 492         minimum => 1,
 493         default => 0,
 494     },
 495     acpi => {
 496         optional => 1,
 497         type => 'boolean',
 498         description => "Enable/disable ACPI.",
 499         default => 1,
 500     },
 501     agent => {
 502         optional => 1,
 503         description => "Enable/disable Qemu GuestAgent and its properties.",
 504         type => 'string',
 505         format => $agent_fmt,
 506     },
 507     kvm => {
 508         optional => 1,
 509         type => 'boolean',
 510         description => "Enable/disable KVM hardware virtualization.",
 511         default => 1,
 512     },
 513     tdf => {
 514         optional => 1,
 515         type => 'boolean',
 516         description => "Enable/disable time drift fix.",
 517         default => 0,
 518     },
 519     localtime => {
 520         optional => 1,
 521         type => 'boolean',
 522         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 523     },
 524     freeze => {
 525         optional => 1,
 526         type => 'boolean',
 527         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 528     },
 529     vga => {
 530         optional => 1,
 531         type => 'string', format => $vga_fmt,
 532         description => "Configure the VGA hardware.",
 533         verbose_description => "Configure the VGA Hardware. If you want to use ".
 534             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 535             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 536             "is 'std' for all OS types besides some Windows versions (XP and " .
 537             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 538             "display server. For win* OS you can select how many independent " .
 539             "displays you want, Linux guests can add displays them self.\n".
 540             "You can also run without any graphic card, using a serial device as terminal.",
 541     },
 542     watchdog => {
 543         optional => 1,
 544         type => 'string', format => 'pve-qm-watchdog',
 545         description => "Create a virtual hardware watchdog device.",
 546         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 547             " (by a guest action), the watchdog must be periodically polled " .
 548             "by an agent inside the guest or else the watchdog will reset " .
 549             "the guest (or execute the respective action specified)",
 550     },
 551     startdate => {
 552         optional => 1,
 553         type => 'string',
 554         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 555         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 556         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 557         default => 'now',
 558     },
 559     startup =>  get_standard_option('pve-startup-order'),
 560     template => {
 561         optional => 1,
 562         type => 'boolean',
 563         description => "Enable/disable Template.",
 564         default => 0,
 565     },
 566     args => {
 567         optional => 1,
 568         type => 'string',
 569         description => "Arbitrary arguments passed to kvm.",
 570         verbose_description => <<EODESCR,
 571 Arbitrary arguments passed to kvm, for example:
 572
 573 args: -no-reboot -no-hpet
 574
 575 NOTE: this option is for experts only.
 576 EODESCR
 577     },
 578     tablet => {
 579         optional => 1,
 580         type => 'boolean',
 581         default => 1,
 582         description => "Enable/disable the USB tablet device.",
 583         verbose_description => "Enable/disable the USB tablet device. This device is " .
 584             "usually needed to allow absolute mouse positioning with VNC. " .
 585             "Else the mouse runs out of sync with normal VNC clients. " .
 586             "If you're running lots of console-only guests on one host, " .
 587             "you may consider disabling this to save some context switches. " .
 588             "This is turned off by default if you use spice (-vga=qxl).",
 589     },
 590     migrate_speed => {
 591         optional => 1,
 592         type => 'integer',
 593         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 594         minimum => 0,
 595         default => 0,
 596     },
 597     migrate_downtime => {
 598         optional => 1,
 599         type => 'number',
 600         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 601         minimum => 0,
 602         default => 0.1,
 603     },
 604     cdrom => {
 605         optional => 1,
 606         type => 'string', format => 'pve-qm-ide',
 607         typetext => '<volume>',
 608         description => "This is an alias for option -ide2",
 609     },
 610     cpu => {
 611         optional => 1,
 612         description => "Emulated CPU type.",
 613         type => 'string',
 614         format => $cpu_fmt,
 615     },
 616     parent => get_standard_option('pve-snapshot-name', {
 617         optional => 1,
 618         description => "Parent snapshot name. This is used internally, and should not be modified.",
 619     }),
 620     snaptime => {
 621         optional => 1,
 622         description => "Timestamp for snapshots.",
 623         type => 'integer',
 624         minimum => 0,
 625     },
 626     vmstate => {
 627         optional => 1,
 628         type => 'string', format => 'pve-volume-id',
 629         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 630     },
 631     vmstatestorage => get_standard_option('pve-storage-id', {
 632         description => "Default storage for VM state volumes/files.",
 633         optional => 1,
 634     }),
 635     runningmachine => get_standard_option('pve-qemu-machine', {
 636         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 637     }),
 638     machine => get_standard_option('pve-qemu-machine'),
 639     arch => {
 640         description => "Virtual processor architecture. Defaults to the host.",
 641         optional => 1,
 642         type => 'string',
 643         enum => [qw(x86_64 aarch64)],
 644     },
 645     smbios1 => {
 646         description => "Specify SMBIOS type 1 fields.",
 647         type => 'string', format => 'pve-qm-smbios1',
 648         maxLength => 512,
 649         optional => 1,
 650     },
 651     protection => {
 652         optional => 1,
 653         type => 'boolean',
 654         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 655         default => 0,
 656     },
 657     bios => {
 658         optional => 1,
 659         type => 'string',
 660         enum => [ qw(seabios ovmf) ],
 661         description => "Select BIOS implementation.",
 662         default => 'seabios',
 663     },
 664     vmgenid => {
 665         type => 'string',
 666         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 667         format_description => 'UUID',
 668         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 669         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 670             " 128-bit integer value identifier to the guest OS. This allows to".
 671             " notify the guest operating system when the virtual machine is".
 672             " executed with a different configuration (e.g. snapshot execution".
 673             " or creation from a template). The guest operating system notices".
 674             " the change, and is then able to react as appropriate by marking".
 675             " its copies of distributed databases as dirty, re-initializing its".
 676             " random number generator, etc.\n".
 677             "Note that auto-creation only works when done throug API/CLI create".
 678             " or update methods, but not when manually editing the config file.",
 679         default => "1 (autogenerated)",
 680         optional => 1,
 681     },
 682     hookscript => {
 683         type => 'string',
 684         format => 'pve-volume-id',
 685         optional => 1,
 686         description => "Script that will be executed during various steps in the vms lifetime.",
 687     },
 688     ivshmem => {
 689         type => 'string',
 690         format => $ivshmem_fmt,
 691         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 692         optional => 1,
 693     },
 694     audio0 => {
 695         type => 'string',
 696         format => $audio_fmt,
 697         description => "Configure a audio device, useful in combination with QXL/Spice.",
 698         optional => 1
 699     },
 700     spice_enhancements => {
 701         type => 'string',
 702         format => $spice_enhancements_fmt,
 703         description => "Configure additional enhancements for SPICE.",
 704         optional => 1
 705     },
 706     tags => {
 707         type => 'string', format => 'pve-tag-list',
 708         description => 'Tags of the VM. This is only meta information.',
 709         optional => 1,
 710     },
 711 };
 712
 713 my $cicustom_fmt = {
 714     meta => {
 715         type => 'string',
 716         optional => 1,
 717         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 718         format => 'pve-volume-id',
 719         format_description => 'volume',
 720     },
 721     network => {
 722         type => 'string',
 723         optional => 1,
 724         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 725         format => 'pve-volume-id',
 726         format_description => 'volume',
 727     },
 728     user => {
 729         type => 'string',
 730         optional => 1,
 731         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 732         format => 'pve-volume-id',
 733         format_description => 'volume',
 734     },
 735 };
 736 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 737
 738 my $confdesc_cloudinit = {
 739     citype => {
 740         optional => 1,
 741         type => 'string',
 742         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 743         enum => ['configdrive2', 'nocloud'],
 744     },
 745     ciuser => {
 746         optional => 1,
 747         type => 'string',
 748         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 749     },
 750     cipassword => {
 751         optional => 1,
 752         type => 'string',
 753         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 754     },
 755     cicustom => {
 756         optional => 1,
 757         type => 'string',
 758         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 759         format => 'pve-qm-cicustom',
 760     },
 761     searchdomain => {
 762         optional => 1,
 763         type => 'string',
 764         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 765     },
 766     nameserver => {
 767         optional => 1,
 768         type => 'string', format => 'address-list',
 769         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 770     },
 771     sshkeys => {
 772         optional => 1,
 773         type => 'string',
 774         format => 'urlencoded',
 775         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 776     },
 777 };
 778
 779 # what about other qemu settings ?
 780 #cpu => 'string',
 781 #machine => 'string',
 782 #fda => 'file',
 783 #fdb => 'file',
 784 #mtdblock => 'file',
 785 #sd => 'file',
 786 #pflash => 'file',
 787 #snapshot => 'bool',
 788 #bootp => 'file',
 789 ##tftp => 'dir',
 790 ##smb => 'dir',
 791 #kernel => 'file',
 792 #append => 'string',
 793 #initrd => 'file',
 794 ##soundhw => 'string',
 795
 796 while (my ($k, $v) = each %$confdesc) {
 797     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 798 }
 799
 800 my $MAX_IDE_DISKS = 4;
 801 my $MAX_SCSI_DISKS = 14;
 802 my $MAX_VIRTIO_DISKS = 16;
 803 my $MAX_SATA_DISKS = 6;
 804 my $MAX_USB_DEVICES = 5;
 805 my $MAX_NETS = 32;
 806 my $MAX_UNUSED_DISKS = 256;
 807 my $MAX_HOSTPCI_DEVICES = 16;
 808 my $MAX_SERIAL_PORTS = 4;
 809 my $MAX_PARALLEL_PORTS = 3;
 810 my $MAX_NUMA = 8;
 811
 812 my $numa_fmt = {
 813     cpus => {
 814         type => "string",
 815         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 816         description => "CPUs accessing this NUMA node.",
 817         format_description => "id[-id];...",
 818     },
 819     memory => {
 820         type => "number",
 821         description => "Amount of memory this NUMA node provides.",
 822         optional => 1,
 823     },
 824     hostnodes => {
 825         type => "string",
 826         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 827         description => "Host NUMA nodes to use.",
 828         format_description => "id[-id];...",
 829         optional => 1,
 830     },
 831     policy => {
 832         type => 'string',
 833         enum => [qw(preferred bind interleave)],
 834         description => "NUMA allocation policy.",
 835         optional => 1,
 836     },
 837 };
 838 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 839 my $numadesc = {
 840     optional => 1,
 841     type => 'string', format => $numa_fmt,
 842     description => "NUMA topology.",
 843 };
 844 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 845
 846 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 847     $confdesc->{"numa$i"} = $numadesc;
 848 }
 849
 850 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 851                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 852                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 853 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 854
 855 my $net_fmt_bridge_descr = <<__EOD__;
 856 Bridge to attach the network device to. The Proxmox VE standard bridge
 857 is called 'vmbr0'.
 858
 859 If you do not specify a bridge, we create a kvm user (NATed) network
 860 device, which provides DHCP and DNS services. The following addresses
 861 are used:
 862
 863  10.0.2.2   Gateway
 864  10.0.2.3   DNS Server
 865  10.0.2.4   SMB Server
 866
 867 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 868 __EOD__
 869
 870 my $net_fmt = {
 871     macaddr  => get_standard_option('mac-addr', {
 872         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 873     }),
 874     model => {
 875         type => 'string',
 876         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 877         enum => $nic_model_list,
 878         default_key => 1,
 879     },
 880     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 881     bridge => {
 882         type => 'string',
 883         description => $net_fmt_bridge_descr,
 884         format_description => 'bridge',
 885         optional => 1,
 886     },
 887     queues => {
 888         type => 'integer',
 889         minimum => 0, maximum => 16,
 890         description => 'Number of packet queues to be used on the device.',
 891         optional => 1,
 892     },
 893     rate => {
 894         type => 'number',
 895         minimum => 0,
 896         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 897         optional => 1,
 898     },
 899     tag => {
 900         type => 'integer',
 901         minimum => 1, maximum => 4094,
 902         description => 'VLAN tag to apply to packets on this interface.',
 903         optional => 1,
 904     },
 905     trunks => {
 906         type => 'string',
 907         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 908         description => 'VLAN trunks to pass through this interface.',
 909         format_description => 'vlanid[;vlanid...]',
 910         optional => 1,
 911     },
 912     firewall => {
 913         type => 'boolean',
 914         description => 'Whether this interface should be protected by the firewall.',
 915         optional => 1,
 916     },
 917     link_down => {
 918         type => 'boolean',
 919         description => 'Whether this interface should be disconnected (like pulling the plug).',
 920         optional => 1,
 921     },
 922 };
 923
 924 my $netdesc = {
 925     optional => 1,
 926     type => 'string', format => $net_fmt,
 927     description => "Specify network devices.",
 928 };
 929
 930 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 931
 932 my $ipconfig_fmt = {
 933     ip => {
 934         type => 'string',
 935         format => 'pve-ipv4-config',
 936         format_description => 'IPv4Format/CIDR',
 937         description => 'IPv4 address in CIDR format.',
 938         optional => 1,
 939         default => 'dhcp',
 940     },
 941     gw => {
 942         type => 'string',
 943         format => 'ipv4',
 944         format_description => 'GatewayIPv4',
 945         description => 'Default gateway for IPv4 traffic.',
 946         optional => 1,
 947         requires => 'ip',
 948     },
 949     ip6 => {
 950         type => 'string',
 951         format => 'pve-ipv6-config',
 952         format_description => 'IPv6Format/CIDR',
 953         description => 'IPv6 address in CIDR format.',
 954         optional => 1,
 955         default => 'dhcp',
 956     },
 957     gw6 => {
 958         type => 'string',
 959         format => 'ipv6',
 960         format_description => 'GatewayIPv6',
 961         description => 'Default gateway for IPv6 traffic.',
 962         optional => 1,
 963         requires => 'ip6',
 964     },
 965 };
 966 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 967 my $ipconfigdesc = {
 968     optional => 1,
 969     type => 'string', format => 'pve-qm-ipconfig',
 970     description => <<'EODESCR',
 971 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 972
 973 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 974
 975 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 976 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 977
 978 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 979 EODESCR
 980 };
 981 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 982
 983 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 984     $confdesc->{"net$i"} = $netdesc;
 985     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 986 }
 987
 988 foreach my $key (keys %$confdesc_cloudinit) {
 989     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 990 }
 991
 992 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 993 sub verify_volume_id_or_qm_path {
 994     my ($volid, $noerr) = @_;
 995
 996     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 997         return $volid;
 998     }
 999
1000     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
1001     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1002     if ($@) {
1003         return undef if $noerr;
1004         die $@;
1005     }
1006     return $volid;
1007 }
1008
1009 my $drivename_hash;
1010
1011 my %drivedesc_base = (
1012     volume => { alias => 'file' },
1013     file => {
1014         type => 'string',
1015         format => 'pve-volume-id-or-qm-path',
1016         default_key => 1,
1017         format_description => 'volume',
1018         description => "The drive's backing volume.",
1019     },
1020     media => {
1021         type => 'string',
1022         enum => [qw(cdrom disk)],
1023         description => "The drive's media type.",
1024         default => 'disk',
1025         optional => 1
1026     },
1027     cyls => {
1028         type => 'integer',
1029         description => "Force the drive's physical geometry to have a specific cylinder count.",
1030         optional => 1
1031     },
1032     heads => {
1033         type => 'integer',
1034         description => "Force the drive's physical geometry to have a specific head count.",
1035         optional => 1
1036     },
1037     secs => {
1038         type => 'integer',
1039         description => "Force the drive's physical geometry to have a specific sector count.",
1040         optional => 1
1041     },
1042     trans => {
1043         type => 'string',
1044         enum => [qw(none lba auto)],
1045         description => "Force disk geometry bios translation mode.",
1046         optional => 1,
1047     },
1048     snapshot => {
1049         type => 'boolean',
1050         description => "Controls qemu's snapshot mode feature."
1051             . " If activated, changes made to the disk are temporary and will"
1052             . " be discarded when the VM is shutdown.",
1053         optional => 1,
1054     },
1055     cache => {
1056         type => 'string',
1057         enum => [qw(none writethrough writeback unsafe directsync)],
1058         description => "The drive's cache mode",
1059         optional => 1,
1060     },
1061     format => get_standard_option('pve-qm-image-format'),
1062     size => {
1063         type => 'string',
1064         format => 'disk-size',
1065         format_description => 'DiskSize',
1066         description => "Disk size. This is purely informational and has no effect.",
1067         optional => 1,
1068     },
1069     backup => {
1070         type => 'boolean',
1071         description => "Whether the drive should be included when making backups.",
1072         optional => 1,
1073     },
1074     replicate => {
1075         type => 'boolean',
1076         description => 'Whether the drive should considered for replication jobs.',
1077         optional => 1,
1078         default => 1,
1079     },
1080     rerror => {
1081         type => 'string',
1082         enum => [qw(ignore report stop)],
1083         description => 'Read error action.',
1084         optional => 1,
1085     },
1086     werror => {
1087         type => 'string',
1088         enum => [qw(enospc ignore report stop)],
1089         description => 'Write error action.',
1090         optional => 1,
1091     },
1092     aio => {
1093         type => 'string',
1094         enum => [qw(native threads)],
1095         description => 'AIO type to use.',
1096         optional => 1,
1097     },
1098     discard => {
1099         type => 'string',
1100         enum => [qw(ignore on)],
1101         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
1102         optional => 1,
1103     },
1104     detect_zeroes => {
1105         type => 'boolean',
1106         description => 'Controls whether to detect and try to optimize writes of zeroes.',
1107         optional => 1,
1108     },
1109     serial => {
1110         type => 'string',
1111         format => 'urlencoded',
1112         format_description => 'serial',
1113         maxLength => 20*3, # *3 since it's %xx url enoded
1114         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
1115         optional => 1,
1116     },
1117     shared => {
1118         type => 'boolean',
1119         description => 'Mark this locally-managed volume as available on all nodes',
1120         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1121         optional => 1,
1122         default => 0,
1123     }
1124 );
1125
1126 my %iothread_fmt = ( iothread => {
1127         type => 'boolean',
1128         description => "Whether to use iothreads for this drive",
1129         optional => 1,
1130 });
1131
1132 my %model_fmt = (
1133     model => {
1134         type => 'string',
1135         format => 'urlencoded',
1136         format_description => 'model',
1137         maxLength => 40*3, # *3 since it's %xx url enoded
1138         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1139         optional => 1,
1140     },
1141 );
1142
1143 my %queues_fmt = (
1144     queues => {
1145         type => 'integer',
1146         description => "Number of queues.",
1147         minimum => 2,
1148         optional => 1
1149     }
1150 );
1151
1152 my %scsiblock_fmt = (
1153     scsiblock => {
1154         type => 'boolean',
1155         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1156         optional => 1,
1157         default => 0,
1158     },
1159 );
1160
1161 my %ssd_fmt = (
1162     ssd => {
1163         type => 'boolean',
1164         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1165         optional => 1,
1166     },
1167 );
1168
1169 my %wwn_fmt = (
1170     wwn => {
1171         type => 'string',
1172         pattern => qr/^(0x)[0-9a-fA-F]{16}/,
1173         format_description => 'wwn',
1174         description => "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.",
1175         optional => 1,
1176     },
1177 );
1178
1179 my $add_throttle_desc = sub {
1180     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1181     my $d = {
1182         type => $type,
1183         format_description => $unit,
1184         description => "Maximum $what in $longunit.",
1185         optional => 1,
1186     };
1187     $d->{minimum} = $minimum if defined($minimum);
1188     $drivedesc_base{$key} = $d;
1189 };
1190 # throughput: (leaky bucket)
1191 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1192 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1193 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1194 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1195 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1196 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1197 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1198 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1199 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1200
1201 # pools: (pool of IO before throttling starts taking effect)
1202 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1203 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1204 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1205 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1206 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1207 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1208
1209 # burst lengths
1210 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1211 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1212 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1213 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1214 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1215 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1216
1217 # legacy support
1218 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1219 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1220 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1221 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1222
1223 my $ide_fmt = {
1224     %drivedesc_base,
1225     %model_fmt,
1226     %ssd_fmt,
1227     %wwn_fmt,
1228 };
1229 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1230
1231 my $idedesc = {
1232     optional => 1,
1233     type => 'string', format => $ide_fmt,
1234     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1235 };
1236 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1237
1238 my $scsi_fmt = {
1239     %drivedesc_base,
1240     %iothread_fmt,
1241     %queues_fmt,
1242     %scsiblock_fmt,
1243     %ssd_fmt,
1244     %wwn_fmt,
1245 };
1246 my $scsidesc = {
1247     optional => 1,
1248     type => 'string', format => $scsi_fmt,
1249     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1250 };
1251 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1252
1253 my $sata_fmt = {
1254     %drivedesc_base,
1255     %ssd_fmt,
1256     %wwn_fmt,
1257 };
1258 my $satadesc = {
1259     optional => 1,
1260     type => 'string', format => $sata_fmt,
1261     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1262 };
1263 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1264
1265 my $virtio_fmt = {
1266     %drivedesc_base,
1267     %iothread_fmt,
1268 };
1269 my $virtiodesc = {
1270     optional => 1,
1271     type => 'string', format => $virtio_fmt,
1272     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1273 };
1274 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1275
1276 my $alldrive_fmt = {
1277     %drivedesc_base,
1278     %iothread_fmt,
1279     %model_fmt,
1280     %queues_fmt,
1281     %scsiblock_fmt,
1282     %ssd_fmt,
1283     %wwn_fmt,
1284 };
1285
1286 my $efidisk_fmt = {
1287     volume => { alias => 'file' },
1288     file => {
1289         type => 'string',
1290         format => 'pve-volume-id-or-qm-path',
1291         default_key => 1,
1292         format_description => 'volume',
1293         description => "The drive's backing volume.",
1294     },
1295     format => get_standard_option('pve-qm-image-format'),
1296     size => {
1297         type => 'string',
1298         format => 'disk-size',
1299         format_description => 'DiskSize',
1300         description => "Disk size. This is purely informational and has no effect.",
1301         optional => 1,
1302     },
1303 };
1304
1305 my $efidisk_desc = {
1306     optional => 1,
1307     type => 'string', format => $efidisk_fmt,
1308     description => "Configure a Disk for storing EFI vars",
1309 };
1310
1311 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1312
1313 my $usb_fmt = {
1314     host => {
1315         default_key => 1,
1316         type => 'string', format => 'pve-qm-usb-device',
1317         format_description => 'HOSTUSBDEVICE|spice',
1318         description => <<EODESCR,
1319 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1320
1321  'bus-port(.port)*' (decimal numbers) or
1322  'vendor_id:product_id' (hexadeciaml numbers) or
1323  'spice'
1324
1325 You can use the 'lsusb -t' command to list existing usb devices.
1326
1327 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1328
1329 The value 'spice' can be used to add a usb redirection devices for spice.
1330 EODESCR
1331     },
1332     usb3 => {
1333         optional => 1,
1334         type => 'boolean',
1335         description => "Specifies whether if given host option is a USB3 device or port.",
1336         default => 0,
1337     },
1338 };
1339
1340 my $usbdesc = {
1341     optional => 1,
1342     type => 'string', format => $usb_fmt,
1343     description => "Configure an USB device (n is 0 to 4).",
1344 };
1345 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1346
1347 my $PCIRE = qr/([a-f0-9]{4}:)?[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1348 my $hostpci_fmt = {
1349     host => {
1350         default_key => 1,
1351         type => 'string',
1352         pattern => qr/$PCIRE(;$PCIRE)*/,
1353         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1354         description => <<EODESCR,
1355 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1356 of PCI virtual functions of the host. HOSTPCIID syntax is:
1357
1358 'bus:dev.func' (hexadecimal numbers)
1359
1360 You can us the 'lspci' command to list existing PCI devices.
1361 EODESCR
1362     },
1363     rombar => {
1364         type => 'boolean',
1365         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1366         optional => 1,
1367         default => 1,
1368     },
1369     romfile => {
1370         type => 'string',
1371         pattern => '[^,;]+',
1372         format_description => 'string',
1373         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1374         optional => 1,
1375     },
1376     pcie => {
1377         type => 'boolean',
1378         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1379         optional => 1,
1380         default => 0,
1381     },
1382     'x-vga' => {
1383         type => 'boolean',
1384         description =>  "Enable vfio-vga device support.",
1385         optional => 1,
1386         default => 0,
1387     },
1388     'mdev' => {
1389         type => 'string',
1390         format_description => 'string',
1391         pattern => '[^/\.:]+',
1392         optional => 1,
1393         description => <<EODESCR
1394 The type of mediated device to use.
1395 An instance of this type will be created on startup of the VM and
1396 will be cleaned up when the VM stops.
1397 EODESCR
1398     }
1399 };
1400 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1401
1402 my $hostpcidesc = {
1403         optional => 1,
1404         type => 'string', format => 'pve-qm-hostpci',
1405         description => "Map host PCI devices into guest.",
1406         verbose_description =>  <<EODESCR,
1407 Map host PCI devices into guest.
1408
1409 NOTE: This option allows direct access to host hardware. So it is no longer
1410 possible to migrate such machines - use with special care.
1411
1412 CAUTION: Experimental! User reported problems with this option.
1413 EODESCR
1414 };
1415 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1416
1417 my $serialdesc = {
1418         optional => 1,
1419         type => 'string',
1420         pattern => '(/dev/.+|socket)',
1421         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1422         verbose_description =>  <<EODESCR,
1423 Create a serial device inside the VM (n is 0 to 3), and pass through a
1424 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1425 host side (use 'qm terminal' to open a terminal connection).
1426
1427 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1428
1429 CAUTION: Experimental! User reported problems with this option.
1430 EODESCR
1431 };
1432
1433 my $paralleldesc= {
1434         optional => 1,
1435         type => 'string',
1436         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1437         description =>  "Map host parallel devices (n is 0 to 2).",
1438         verbose_description =>  <<EODESCR,
1439 Map host parallel devices (n is 0 to 2).
1440
1441 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1442
1443 CAUTION: Experimental! User reported problems with this option.
1444 EODESCR
1445 };
1446
1447 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1448     $confdesc->{"parallel$i"} = $paralleldesc;
1449 }
1450
1451 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1452     $confdesc->{"serial$i"} = $serialdesc;
1453 }
1454
1455 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1456     $confdesc->{"hostpci$i"} = $hostpcidesc;
1457 }
1458
1459 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1460     $drivename_hash->{"ide$i"} = 1;
1461     $confdesc->{"ide$i"} = $idedesc;
1462 }
1463
1464 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1465     $drivename_hash->{"sata$i"} = 1;
1466     $confdesc->{"sata$i"} = $satadesc;
1467 }
1468
1469 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1470     $drivename_hash->{"scsi$i"} = 1;
1471     $confdesc->{"scsi$i"} = $scsidesc ;
1472 }
1473
1474 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1475     $drivename_hash->{"virtio$i"} = 1;
1476     $confdesc->{"virtio$i"} = $virtiodesc;
1477 }
1478
1479 $drivename_hash->{efidisk0} = 1;
1480 $confdesc->{efidisk0} = $efidisk_desc;
1481
1482 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1483     $confdesc->{"usb$i"} = $usbdesc;
1484 }
1485
1486 my $unuseddesc = {
1487     optional => 1,
1488     type => 'string', format => 'pve-volume-id',
1489     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1490 };
1491
1492 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1493     $confdesc->{"unused$i"} = $unuseddesc;
1494 }
1495
1496 my $kvm_api_version = 0;
1497
1498 sub kvm_version {
1499     return $kvm_api_version if $kvm_api_version;
1500
1501     open my $fh, '<', '/dev/kvm'
1502         or return undef;
1503
1504     # 0xae00 => KVM_GET_API_VERSION
1505     $kvm_api_version = ioctl($fh, 0xae00, 0);
1506
1507     return $kvm_api_version;
1508 }
1509
1510 my $kvm_user_version = {};
1511 my $kvm_mtime = {};
1512
1513 sub kvm_user_version {
1514     my ($binary) = @_;
1515
1516     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1517     my $st = stat($binary);
1518
1519     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1520     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1521         $cachedmtime == $st->mtime;
1522
1523     $kvm_user_version->{$binary} = 'unknown';
1524     $kvm_mtime->{$binary} = $st->mtime;
1525
1526     my $code = sub {
1527         my $line = shift;
1528         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1529             $kvm_user_version->{$binary} = $2;
1530         }
1531     };
1532
1533     eval { run_command([$binary, '--version'], outfunc => $code); };
1534     warn $@ if $@;
1535
1536     return $kvm_user_version->{$binary};
1537
1538 }
1539
1540 sub kernel_has_vhost_net {
1541     return -c '/dev/vhost-net';
1542 }
1543
1544 sub valid_drive_names {
1545     # order is important - used to autoselect boot disk
1546     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1547             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1548             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1549             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1550             'efidisk0');
1551 }
1552
1553 sub is_valid_drivename {
1554     my $dev = shift;
1555
1556     return defined($drivename_hash->{$dev});
1557 }
1558
1559 sub option_exists {
1560     my $key = shift;
1561     return defined($confdesc->{$key});
1562 }
1563
1564 my $cdrom_path;
1565 sub get_cdrom_path {
1566
1567     return  $cdrom_path if $cdrom_path;
1568
1569     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1570     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1571     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1572 }
1573
1574 sub get_iso_path {
1575     my ($storecfg, $vmid, $cdrom) = @_;
1576
1577     if ($cdrom eq 'cdrom') {
1578         return get_cdrom_path();
1579     } elsif ($cdrom eq 'none') {
1580         return '';
1581     } elsif ($cdrom =~ m|^/|) {
1582         return $cdrom;
1583     } else {
1584         return PVE::Storage::path($storecfg, $cdrom);
1585     }
1586 }
1587
1588 # try to convert old style file names to volume IDs
1589 sub filename_to_volume_id {
1590     my ($vmid, $file, $media) = @_;
1591
1592      if (!($file eq 'none' || $file eq 'cdrom' ||
1593           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1594
1595         return undef if $file =~ m|/|;
1596
1597         if ($media && $media eq 'cdrom') {
1598             $file = "local:iso/$file";
1599         } else {
1600             $file = "local:$vmid/$file";
1601         }
1602     }
1603
1604     return $file;
1605 }
1606
1607 sub verify_media_type {
1608     my ($opt, $vtype, $media) = @_;
1609
1610     return if !$media;
1611
1612     my $etype;
1613     if ($media eq 'disk') {
1614         $etype = 'images';
1615     } elsif ($media eq 'cdrom') {
1616         $etype = 'iso';
1617     } else {
1618         die "internal error";
1619     }
1620
1621     return if ($vtype eq $etype);
1622
1623     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1624 }
1625
1626 sub cleanup_drive_path {
1627     my ($opt, $storecfg, $drive) = @_;
1628
1629     # try to convert filesystem paths to volume IDs
1630
1631     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1632         ($drive->{file} !~ m|^/dev/.+|) &&
1633         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1634         ($drive->{file} !~ m/^\d+$/)) {
1635         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1636         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1637         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1638         verify_media_type($opt, $vtype, $drive->{media});
1639         $drive->{file} = $volid;
1640     }
1641
1642     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1643 }
1644
1645 sub parse_hotplug_features {
1646     my ($data) = @_;
1647
1648     my $res = {};
1649
1650     return $res if $data eq '0';
1651
1652     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1653
1654     foreach my $feature (PVE::Tools::split_list($data)) {
1655         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1656             $res->{$1} = 1;
1657         } else {
1658             die "invalid hotplug feature '$feature'\n";
1659         }
1660     }
1661     return $res;
1662 }
1663
1664 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1665 sub pve_verify_hotplug_features {
1666     my ($value, $noerr) = @_;
1667
1668     return $value if parse_hotplug_features($value);
1669
1670     return undef if $noerr;
1671
1672     die "unable to parse hotplug option\n";
1673 }
1674
1675 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1676 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1677 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1678 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1679 #        [,iothread=on][,serial=serial][,model=model]
1680
1681 sub parse_drive {
1682     my ($key, $data) = @_;
1683
1684     my ($interface, $index);
1685
1686     if ($key =~ m/^([^\d]+)(\d+)$/) {
1687         $interface = $1;
1688         $index = $2;
1689     } else {
1690         return undef;
1691     }
1692
1693     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1694                                      : $confdesc->{$key}->{format};
1695     if (!$desc) {
1696         warn "invalid drive key: $key\n";
1697         return undef;
1698     }
1699     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1700     return undef if !$res;
1701     $res->{interface} = $interface;
1702     $res->{index} = $index;
1703
1704     my $error = 0;
1705     foreach my $opt (qw(bps bps_rd bps_wr)) {
1706         if (my $bps = defined(delete $res->{$opt})) {
1707             if (defined($res->{"m$opt"})) {
1708                 warn "both $opt and m$opt specified\n";
1709                 ++$error;
1710                 next;
1711             }
1712             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1713         }
1714     }
1715
1716     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1717     for my $requirement (
1718         [mbps_max => 'mbps'],
1719         [mbps_rd_max => 'mbps_rd'],
1720         [mbps_wr_max => 'mbps_wr'],
1721         [miops_max => 'miops'],
1722         [miops_rd_max => 'miops_rd'],
1723         [miops_wr_max => 'miops_wr'],
1724         [bps_max_length => 'mbps_max'],
1725         [bps_rd_max_length => 'mbps_rd_max'],
1726         [bps_wr_max_length => 'mbps_wr_max'],
1727         [iops_max_length => 'iops_max'],
1728         [iops_rd_max_length => 'iops_rd_max'],
1729         [iops_wr_max_length => 'iops_wr_max']) {
1730         my ($option, $requires) = @$requirement;
1731         if ($res->{$option} && !$res->{$requires}) {
1732             warn "$option requires $requires\n";
1733             ++$error;
1734         }
1735     }
1736
1737     return undef if $error;
1738
1739     return undef if $res->{mbps_rd} && $res->{mbps};
1740     return undef if $res->{mbps_wr} && $res->{mbps};
1741     return undef if $res->{iops_rd} && $res->{iops};
1742     return undef if $res->{iops_wr} && $res->{iops};
1743
1744     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1745         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1746         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1747         return undef if $res->{interface} eq 'virtio';
1748     }
1749
1750     if (my $size = $res->{size}) {
1751         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1752     }
1753
1754     return $res;
1755 }
1756
1757 sub print_drive {
1758     my ($drive) = @_;
1759     my $data = { %$drive };
1760     delete $data->{$_} for qw(index interface);
1761     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1762 }
1763
1764 sub scsi_inquiry {
1765     my($fh, $noerr) = @_;
1766
1767     my $SG_IO = 0x2285;
1768     my $SG_GET_VERSION_NUM = 0x2282;
1769
1770     my $versionbuf = "\x00" x 8;
1771     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1772     if (!$ret) {
1773         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1774         return undef;
1775     }
1776     my $version = unpack("I", $versionbuf);
1777     if ($version < 30000) {
1778         die "scsi generic interface too old\n"  if !$noerr;
1779         return undef;
1780     }
1781
1782     my $buf = "\x00" x 36;
1783     my $sensebuf = "\x00" x 8;
1784     my $cmd = pack("C x3 C x1", 0x12, 36);
1785
1786     # see /usr/include/scsi/sg.h
1787     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1788
1789     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1790                       length($sensebuf), 0, length($buf), $buf,
1791                       $cmd, $sensebuf, 6000);
1792
1793     $ret = ioctl($fh, $SG_IO, $packet);
1794     if (!$ret) {
1795         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1796         return undef;
1797     }
1798
1799     my @res = unpack($sg_io_hdr_t, $packet);
1800     if ($res[17] || $res[18]) {
1801         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1802         return undef;
1803     }
1804
1805     my $res = {};
1806     (my $byte0, my $byte1, $res->{vendor},
1807      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1808
1809     $res->{removable} = $byte1 & 128 ? 1 : 0;
1810     $res->{type} = $byte0 & 31;
1811
1812     return $res;
1813 }
1814
1815 sub path_is_scsi {
1816     my ($path) = @_;
1817
1818     my $fh = IO::File->new("+<$path") || return undef;
1819     my $res = scsi_inquiry($fh, 1);
1820     close($fh);
1821
1822     return $res;
1823 }
1824
1825 sub print_tabletdevice_full {
1826     my ($conf, $arch) = @_;
1827
1828     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1829
1830     # we use uhci for old VMs because tablet driver was buggy in older qemu
1831     my $usbbus;
1832     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1833         $usbbus = 'ehci';
1834     } else {
1835         $usbbus = 'uhci';
1836     }
1837
1838     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1839 }
1840
1841 sub print_keyboarddevice_full {
1842     my ($conf, $arch, $machine) = @_;
1843
1844     return undef if $arch ne 'aarch64';
1845
1846     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1847 }
1848
1849 sub print_drivedevice_full {
1850     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1851
1852     my $device = '';
1853     my $maxdev = 0;
1854
1855     if ($drive->{interface} eq 'virtio') {
1856         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1857         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1858         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1859     } elsif ($drive->{interface} eq 'scsi') {
1860
1861         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1862         my $unit = $drive->{index} % $maxdev;
1863         my $devicetype = 'hd';
1864         my $path = '';
1865         if (drive_is_cdrom($drive)) {
1866             $devicetype = 'cd';
1867         } else {
1868             if ($drive->{file} =~ m|^/|) {
1869                 $path = $drive->{file};
1870                 if (my $info = path_is_scsi($path)) {
1871                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1872                         $devicetype = 'block';
1873                     } elsif ($info->{type} == 1) { # tape
1874                         $devicetype = 'generic';
1875                     }
1876                 }
1877             } else {
1878                  $path = PVE::Storage::path($storecfg, $drive->{file});
1879             }
1880
1881             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1882             my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
1883             if ($path =~ m/^iscsi\:\/\// &&
1884                !min_version($version, 4, 1)) {
1885                 $devicetype = 'generic';
1886             }
1887         }
1888
1889         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1890             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1891         } else {
1892             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1893         }
1894
1895         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1896             $device .= ",rotation_rate=1";
1897         }
1898         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1899
1900     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1901         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1902         my $controller = int($drive->{index} / $maxdev);
1903         my $unit = $drive->{index} % $maxdev;
1904         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1905
1906         $device = "ide-$devicetype";
1907         if ($drive->{interface} eq 'ide') {
1908             $device .= ",bus=ide.$controller,unit=$unit";
1909         } else {
1910             $device .= ",bus=ahci$controller.$unit";
1911         }
1912         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1913
1914         if ($devicetype eq 'hd') {
1915             if (my $model = $drive->{model}) {
1916                 $model = URI::Escape::uri_unescape($model);
1917                 $device .= ",model=$model";
1918             }
1919             if ($drive->{ssd}) {
1920                 $device .= ",rotation_rate=1";
1921             }
1922         }
1923         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1924     } elsif ($drive->{interface} eq 'usb') {
1925         die "implement me";
1926         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1927     } else {
1928         die "unsupported interface type";
1929     }
1930
1931     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1932
1933     if (my $serial = $drive->{serial}) {
1934         $serial = URI::Escape::uri_unescape($serial);
1935         $device .= ",serial=$serial";
1936     }
1937
1938
1939     return $device;
1940 }
1941
1942 sub get_initiator_name {
1943     my $initiator;
1944
1945     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1946     while (defined(my $line = <$fh>)) {
1947         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1948         $initiator = $1;
1949         last;
1950     }
1951     $fh->close();
1952
1953     return $initiator;
1954 }
1955
1956 sub print_drive_full {
1957     my ($storecfg, $vmid, $drive) = @_;
1958
1959     my $path;
1960     my $volid = $drive->{file};
1961     my $format;
1962
1963     if (drive_is_cdrom($drive)) {
1964         $path = get_iso_path($storecfg, $vmid, $volid);
1965     } else {
1966         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1967         if ($storeid) {
1968             $path = PVE::Storage::path($storecfg, $volid);
1969             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1970             $format = qemu_img_format($scfg, $volname);
1971         } else {
1972             $path = $volid;
1973             $format = "raw";
1974         }
1975    }
1976
1977     my $opts = '';
1978     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1979     foreach my $o (@qemu_drive_options) {
1980         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1981     }
1982
1983     # snapshot only accepts on|off
1984     if (defined($drive->{snapshot})) {
1985         my $v = $drive->{snapshot} ? 'on' : 'off';
1986         $opts .= ",snapshot=$v";
1987     }
1988
1989     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1990         my ($dir, $qmpname) = @$type;
1991         if (my $v = $drive->{"mbps$dir"}) {
1992             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1993         }
1994         if (my $v = $drive->{"mbps${dir}_max"}) {
1995             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1996         }
1997         if (my $v = $drive->{"bps${dir}_max_length"}) {
1998             $opts .= ",throttling.bps$qmpname-max-length=$v";
1999         }
2000         if (my $v = $drive->{"iops${dir}"}) {
2001             $opts .= ",throttling.iops$qmpname=$v";
2002         }
2003         if (my $v = $drive->{"iops${dir}_max"}) {
2004             $opts .= ",throttling.iops$qmpname-max=$v";
2005         }
2006         if (my $v = $drive->{"iops${dir}_max_length"}) {
2007             $opts .= ",throttling.iops$qmpname-max-length=$v";
2008         }
2009     }
2010
2011     $opts .= ",format=$format" if $format && !$drive->{format};
2012
2013     my $cache_direct = 0;
2014
2015     if (my $cache = $drive->{cache}) {
2016         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
2017     } elsif (!drive_is_cdrom($drive)) {
2018         $opts .= ",cache=none";
2019         $cache_direct = 1;
2020     }
2021
2022     # aio native works only with O_DIRECT
2023     if (!$drive->{aio}) {
2024         if($cache_direct) {
2025             $opts .= ",aio=native";
2026         } else {
2027             $opts .= ",aio=threads";
2028         }
2029     }
2030
2031     if (!drive_is_cdrom($drive)) {
2032         my $detectzeroes;
2033         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
2034             $detectzeroes = 'off';
2035         } elsif ($drive->{discard}) {
2036             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
2037         } else {
2038             # This used to be our default with discard not being specified:
2039             $detectzeroes = 'on';
2040         }
2041         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
2042     }
2043
2044     my $pathinfo = $path ? "file=$path," : '';
2045
2046     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
2047 }
2048
2049 sub print_netdevice_full {
2050     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
2051
2052     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2053
2054     my $device = $net->{model};
2055     if ($net->{model} eq 'virtio') {
2056          $device = 'virtio-net-pci';
2057      };
2058
2059     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
2060     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
2061     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
2062         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
2063         my $vectors = $net->{queues} * 2 + 2;
2064         $tmpstr .= ",vectors=$vectors,mq=on";
2065     }
2066     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
2067
2068     if ($use_old_bios_files) {
2069         my $romfile;
2070         if ($device eq 'virtio-net-pci') {
2071             $romfile = 'pxe-virtio.rom';
2072         } elsif ($device eq 'e1000') {
2073             $romfile = 'pxe-e1000.rom';
2074         } elsif ($device eq 'ne2k') {
2075             $romfile = 'pxe-ne2k_pci.rom';
2076         } elsif ($device eq 'pcnet') {
2077             $romfile = 'pxe-pcnet.rom';
2078         } elsif ($device eq 'rtl8139') {
2079             $romfile = 'pxe-rtl8139.rom';
2080         }
2081         $tmpstr .= ",romfile=$romfile" if $romfile;
2082     }
2083
2084     return $tmpstr;
2085 }
2086
2087 sub print_netdev_full {
2088     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
2089
2090     my $i = '';
2091     if ($netid =~ m/^net(\d+)$/) {
2092         $i = int($1);
2093     }
2094
2095     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
2096
2097     my $ifname = "tap${vmid}i$i";
2098
2099     # kvm uses TUNSETIFF ioctl, and that limits ifname length
2100     die "interface name '$ifname' is too long (max 15 character)\n"
2101         if length($ifname) >= 16;
2102
2103     my $vhostparam = '';
2104     if (is_native($arch)) {
2105         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
2106     }
2107
2108     my $vmname = $conf->{name} || "vm$vmid";
2109
2110     my $netdev = "";
2111     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
2112
2113     if ($net->{bridge}) {
2114         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
2115     } else {
2116         $netdev = "type=user,id=$netid,hostname=$vmname";
2117     }
2118
2119     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
2120
2121     return $netdev;
2122 }
2123
2124
2125 sub print_cpu_device {
2126     my ($conf, $id) = @_;
2127
2128     my $kvm = $conf->{kvm} // 1;
2129     my $cpu = $kvm ? "kvm64" : "qemu64";
2130     if (my $cputype = $conf->{cpu}) {
2131         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
2132             or die "Cannot parse cpu description: $cputype\n";
2133         $cpu = $cpuconf->{cputype};
2134     }
2135
2136     my $cores = $conf->{cores} || 1;
2137
2138     my $current_core = ($id - 1) % $cores;
2139     my $current_socket = int(($id - 1 - $current_core)/$cores);
2140
2141     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2142 }
2143
2144 my $vga_map = {
2145     'cirrus' => 'cirrus-vga',
2146     'std' => 'VGA',
2147     'vmware' => 'vmware-svga',
2148     'virtio' => 'virtio-vga',
2149 };
2150
2151 sub print_vga_device {
2152     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
2153
2154     my $type = $vga_map->{$vga->{type}};
2155     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2156         $type = 'virtio-gpu';
2157     }
2158     my $vgamem_mb = $vga->{memory};
2159
2160     my $max_outputs = '';
2161     if ($qxlnum) {
2162         $type = $id ? 'qxl' : 'qxl-vga';
2163
2164         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
2165             # set max outputs so linux can have up to 4 qxl displays with one device
2166             if (min_version($machine_version, 4, 1)) {
2167                 $max_outputs = ",max_outputs=4";
2168             }
2169         }
2170     }
2171
2172     die "no devicetype for $vga->{type}\n" if !$type;
2173
2174     my $memory = "";
2175     if ($vgamem_mb) {
2176         if ($vga->{type} eq 'virtio') {
2177             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2178             $memory = ",max_hostmem=$bytes";
2179         } elsif ($qxlnum) {
2180             # from https://www.spice-space.org/multiple-monitors.html
2181             $memory = ",vgamem_mb=$vga->{memory}";
2182             my $ram = $vgamem_mb * 4;
2183             my $vram = $vgamem_mb * 2;
2184             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2185         } else {
2186             $memory = ",vgamem_mb=$vga->{memory}";
2187         }
2188     } elsif ($qxlnum && $id) {
2189         $memory = ",ram_size=67108864,vram_size=33554432";
2190     }
2191
2192     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2193     my $vgaid = "vga" . ($id // '');
2194     my $pciaddr;
2195
2196     if ($q35 && $vgaid eq 'vga') {
2197         # the first display uses pcie.0 bus on q35 machines
2198         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2199     } else {
2200         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2201     }
2202
2203     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}";
2204 }
2205
2206 sub drive_is_cloudinit {
2207     my ($drive) = @_;
2208     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2209 }
2210
2211 sub drive_is_cdrom {
2212     my ($drive, $exclude_cloudinit) = @_;
2213
2214     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2215
2216     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2217
2218 }
2219
2220 sub parse_number_sets {
2221     my ($set) = @_;
2222     my $res = [];
2223     foreach my $part (split(/;/, $set)) {
2224         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2225             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2226             push @$res, [ $1, $2 ];
2227         } else {
2228             die "invalid range: $part\n";
2229         }
2230     }
2231     return $res;
2232 }
2233
2234 sub parse_numa {
2235     my ($data) = @_;
2236
2237     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2238     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2239     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2240     return $res;
2241 }
2242
2243 sub parse_hostpci {
2244     my ($value) = @_;
2245
2246     return undef if !$value;
2247
2248     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2249
2250     my @idlist = split(/;/, $res->{host});
2251     delete $res->{host};
2252     foreach my $id (@idlist) {
2253         my $devs = PVE::SysFSTools::lspci($id);
2254         die "no PCI device found for '$id'\n" if !scalar(@$devs);
2255         push @{$res->{pciid}}, @$devs;
2256     }
2257     return $res;
2258 }
2259
2260 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2261 sub parse_net {
2262     my ($data) = @_;
2263
2264     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2265     if ($@) {
2266         warn $@;
2267         return undef;
2268     }
2269     if (!defined($res->{macaddr})) {
2270         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2271         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2272     }
2273     return $res;
2274 }
2275
2276 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2277 sub parse_ipconfig {
2278     my ($data) = @_;
2279
2280     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2281     if ($@) {
2282         warn $@;
2283         return undef;
2284     }
2285
2286     if ($res->{gw} && !$res->{ip}) {
2287         warn 'gateway specified without specifying an IP address';
2288         return undef;
2289     }
2290     if ($res->{gw6} && !$res->{ip6}) {
2291         warn 'IPv6 gateway specified without specifying an IPv6 address';
2292         return undef;
2293     }
2294     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2295         warn 'gateway specified together with DHCP';
2296         return undef;
2297     }
2298     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2299         # gw6 + auto/dhcp
2300         warn "IPv6 gateway specified together with $res->{ip6} address";
2301         return undef;
2302     }
2303
2304     if (!$res->{ip} && !$res->{ip6}) {
2305         return { ip => 'dhcp', ip6 => 'dhcp' };
2306     }
2307
2308     return $res;
2309 }
2310
2311 sub print_net {
2312     my $net = shift;
2313
2314     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2315 }
2316
2317 sub add_random_macs {
2318     my ($settings) = @_;
2319
2320     foreach my $opt (keys %$settings) {
2321         next if $opt !~ m/^net(\d+)$/;
2322         my $net = parse_net($settings->{$opt});
2323         next if !$net;
2324         $settings->{$opt} = print_net($net);
2325     }
2326 }
2327
2328 sub vm_is_volid_owner {
2329     my ($storecfg, $vmid, $volid) = @_;
2330
2331     if ($volid !~  m|^/|) {
2332         my ($path, $owner);
2333         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2334         if ($owner && ($owner == $vmid)) {
2335             return 1;
2336         }
2337     }
2338
2339     return undef;
2340 }
2341
2342 sub vmconfig_register_unused_drive {
2343     my ($storecfg, $vmid, $conf, $drive) = @_;
2344
2345     if (drive_is_cloudinit($drive)) {
2346         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2347         warn $@ if $@;
2348     } elsif (!drive_is_cdrom($drive)) {
2349         my $volid = $drive->{file};
2350         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2351             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2352         }
2353     }
2354 }
2355
2356 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2357 my $smbios1_fmt = {
2358     uuid => {
2359         type => 'string',
2360         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2361         format_description => 'UUID',
2362         description => "Set SMBIOS1 UUID.",
2363         optional => 1,
2364     },
2365     version => {
2366         type => 'string',
2367         pattern => '[A-Za-z0-9+\/]+={0,2}',
2368         format_description => 'Base64 encoded string',
2369         description => "Set SMBIOS1 version.",
2370         optional => 1,
2371     },
2372     serial => {
2373         type => 'string',
2374         pattern => '[A-Za-z0-9+\/]+={0,2}',
2375         format_description => 'Base64 encoded string',
2376         description => "Set SMBIOS1 serial number.",
2377         optional => 1,
2378     },
2379     manufacturer => {
2380         type => 'string',
2381         pattern => '[A-Za-z0-9+\/]+={0,2}',
2382         format_description => 'Base64 encoded string',
2383         description => "Set SMBIOS1 manufacturer.",
2384         optional => 1,
2385     },
2386     product => {
2387         type => 'string',
2388         pattern => '[A-Za-z0-9+\/]+={0,2}',
2389         format_description => 'Base64 encoded string',
2390         description => "Set SMBIOS1 product ID.",
2391         optional => 1,
2392     },
2393     sku => {
2394         type => 'string',
2395         pattern => '[A-Za-z0-9+\/]+={0,2}',
2396         format_description => 'Base64 encoded string',
2397         description => "Set SMBIOS1 SKU string.",
2398         optional => 1,
2399     },
2400     family => {
2401         type => 'string',
2402         pattern => '[A-Za-z0-9+\/]+={0,2}',
2403         format_description => 'Base64 encoded string',
2404         description => "Set SMBIOS1 family string.",
2405         optional => 1,
2406     },
2407     base64 => {
2408         type => 'boolean',
2409         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2410         optional => 1,
2411     },
2412 };
2413
2414 sub parse_smbios1 {
2415     my ($data) = @_;
2416
2417     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2418     warn $@ if $@;
2419     return $res;
2420 }
2421
2422 sub print_smbios1 {
2423     my ($smbios1) = @_;
2424     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2425 }
2426
2427 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2428
2429 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2430 sub verify_bootdisk {
2431     my ($value, $noerr) = @_;
2432
2433     return $value if is_valid_drivename($value);
2434
2435     return undef if $noerr;
2436
2437     die "invalid boot disk '$value'\n";
2438 }
2439
2440 sub parse_watchdog {
2441     my ($value) = @_;
2442
2443     return undef if !$value;
2444
2445     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2446     warn $@ if $@;
2447     return $res;
2448 }
2449
2450 sub parse_guest_agent {
2451     my ($value) = @_;
2452
2453     return {} if !defined($value->{agent});
2454
2455     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2456     warn $@ if $@;
2457
2458     # if the agent is disabled ignore the other potentially set properties
2459     return {} if !$res->{enabled};
2460     return $res;
2461 }
2462
2463 sub parse_vga {
2464     my ($value) = @_;
2465
2466     return {} if !$value;
2467     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2468     warn $@ if $@;
2469     return $res;
2470 }
2471
2472 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2473 sub verify_usb_device {
2474     my ($value, $noerr) = @_;
2475
2476     return $value if parse_usb_device($value);
2477
2478     return undef if $noerr;
2479
2480     die "unable to parse usb device\n";
2481 }
2482
2483 # add JSON properties for create and set function
2484 sub json_config_properties {
2485     my $prop = shift;
2486
2487     foreach my $opt (keys %$confdesc) {
2488         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2489         $prop->{$opt} = $confdesc->{$opt};
2490     }
2491
2492     return $prop;
2493 }
2494
2495 # return copy of $confdesc_cloudinit to generate documentation
2496 sub cloudinit_config_properties {
2497
2498     return dclone($confdesc_cloudinit);
2499 }
2500
2501 sub check_type {
2502     my ($key, $value) = @_;
2503
2504     die "unknown setting '$key'\n" if !$confdesc->{$key};
2505
2506     my $type = $confdesc->{$key}->{type};
2507
2508     if (!defined($value)) {
2509         die "got undefined value\n";
2510     }
2511
2512     if ($value =~ m/[\n\r]/) {
2513         die "property contains a line feed\n";
2514     }
2515
2516     if ($type eq 'boolean') {
2517         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2518         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2519         die "type check ('boolean') failed - got '$value'\n";
2520     } elsif ($type eq 'integer') {
2521         return int($1) if $value =~ m/^(\d+)$/;
2522         die "type check ('integer') failed - got '$value'\n";
2523     } elsif ($type eq 'number') {
2524         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2525         die "type check ('number') failed - got '$value'\n";
2526     } elsif ($type eq 'string') {
2527         if (my $fmt = $confdesc->{$key}->{format}) {
2528             PVE::JSONSchema::check_format($fmt, $value);
2529             return $value;
2530         }
2531         $value =~ s/^\"(.*)\"$/$1/;
2532         return $value;
2533     } else {
2534         die "internal error"
2535     }
2536 }
2537
2538 sub destroy_vm {
2539     my ($storecfg, $vmid, $skiplock, $replacement_conf) = @_;
2540
2541     my $conf = PVE::QemuConfig->load_config($vmid);
2542
2543     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2544
2545     if ($conf->{template}) {
2546         # check if any base image is still used by a linked clone
2547         foreach_drive($conf, sub {
2548                 my ($ds, $drive) = @_;
2549                 return if drive_is_cdrom($drive);
2550
2551                 my $volid = $drive->{file};
2552                 return if !$volid || $volid =~ m|^/|;
2553
2554                 die "base volume '$volid' is still in use by linked cloned\n"
2555                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2556
2557         });
2558     }
2559
2560     # only remove disks owned by this VM
2561     foreach_drive($conf, sub {
2562         my ($ds, $drive) = @_;
2563         return if drive_is_cdrom($drive, 1);
2564
2565         my $volid = $drive->{file};
2566         return if !$volid || $volid =~ m|^/|;
2567
2568         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2569         return if !$path || !$owner || ($owner != $vmid);
2570
2571         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2572         warn "Could not remove disk '$volid', check manually: $@" if $@;
2573     });
2574
2575     # also remove unused disk
2576     my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2577     PVE::Storage::foreach_volid($vmdisks, sub {
2578         my ($volid, $sid, $volname, $d) = @_;
2579         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2580         warn $@ if $@;
2581     });
2582
2583     if (defined $replacement_conf) {
2584         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2585     } else {
2586         PVE::QemuConfig->destroy_config($vmid);
2587     }
2588 }
2589
2590 sub parse_vm_config {
2591     my ($filename, $raw) = @_;
2592
2593     return undef if !defined($raw);
2594
2595     my $res = {
2596         digest => Digest::SHA::sha1_hex($raw),
2597         snapshots => {},
2598         pending => {},
2599     };
2600
2601     $filename =~ m|/qemu-server/(\d+)\.conf$|
2602         || die "got strange filename '$filename'";
2603
2604     my $vmid = $1;
2605
2606     my $conf = $res;
2607     my $descr;
2608     my $section = '';
2609
2610     my @lines = split(/\n/, $raw);
2611     foreach my $line (@lines) {
2612         next if $line =~ m/^\s*$/;
2613
2614         if ($line =~ m/^\[PENDING\]\s*$/i) {
2615             $section = 'pending';
2616             if (defined($descr)) {
2617                 $descr =~ s/\s+$//;
2618                 $conf->{description} = $descr;
2619             }
2620             $descr = undef;
2621             $conf = $res->{$section} = {};
2622             next;
2623
2624         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2625             $section = $1;
2626             if (defined($descr)) {
2627                 $descr =~ s/\s+$//;
2628                 $conf->{description} = $descr;
2629             }
2630             $descr = undef;
2631             $conf = $res->{snapshots}->{$section} = {};
2632             next;
2633         }
2634
2635         if ($line =~ m/^\#(.*)\s*$/) {
2636             $descr = '' if !defined($descr);
2637             $descr .= PVE::Tools::decode_text($1) . "\n";
2638             next;
2639         }
2640
2641         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2642             $descr = '' if !defined($descr);
2643             $descr .= PVE::Tools::decode_text($2);
2644         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2645             $conf->{snapstate} = $1;
2646         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2647             my $key = $1;
2648             my $value = $2;
2649             $conf->{$key} = $value;
2650         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2651             my $value = $1;
2652             if ($section eq 'pending') {
2653                 $conf->{delete} = $value; # we parse this later
2654             } else {
2655                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2656             }
2657         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2658             my $key = $1;
2659             my $value = $2;
2660             eval { $value = check_type($key, $value); };
2661             if ($@) {
2662                 warn "vm $vmid - unable to parse value of '$key' - $@";
2663             } else {
2664                 $key = 'ide2' if $key eq 'cdrom';
2665                 my $fmt = $confdesc->{$key}->{format};
2666                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2667                     my $v = parse_drive($key, $value);
2668                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2669                         $v->{file} = $volid;
2670                         $value = print_drive($v);
2671                     } else {
2672                         warn "vm $vmid - unable to parse value of '$key'\n";
2673                         next;
2674                     }
2675                 }
2676
2677                 $conf->{$key} = $value;
2678             }
2679         }
2680     }
2681
2682     if (defined($descr)) {
2683         $descr =~ s/\s+$//;
2684         $conf->{description} = $descr;
2685     }
2686     delete $res->{snapstate}; # just to be sure
2687
2688     return $res;
2689 }
2690
2691 sub write_vm_config {
2692     my ($filename, $conf) = @_;
2693
2694     delete $conf->{snapstate}; # just to be sure
2695
2696     if ($conf->{cdrom}) {
2697         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2698         $conf->{ide2} = $conf->{cdrom};
2699         delete $conf->{cdrom};
2700     }
2701
2702     # we do not use 'smp' any longer
2703     if ($conf->{sockets}) {
2704         delete $conf->{smp};
2705     } elsif ($conf->{smp}) {
2706         $conf->{sockets} = $conf->{smp};
2707         delete $conf->{cores};
2708         delete $conf->{smp};
2709     }
2710
2711     my $used_volids = {};
2712
2713     my $cleanup_config = sub {
2714         my ($cref, $pending, $snapname) = @_;
2715
2716         foreach my $key (keys %$cref) {
2717             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2718                 $key eq 'snapstate' || $key eq 'pending';
2719             my $value = $cref->{$key};
2720             if ($key eq 'delete') {
2721                 die "propertry 'delete' is only allowed in [PENDING]\n"
2722                     if !$pending;
2723                 # fixme: check syntax?
2724                 next;
2725             }
2726             eval { $value = check_type($key, $value); };
2727             die "unable to parse value of '$key' - $@" if $@;
2728
2729             $cref->{$key} = $value;
2730
2731             if (!$snapname && is_valid_drivename($key)) {
2732                 my $drive = parse_drive($key, $value);
2733                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2734             }
2735         }
2736     };
2737
2738     &$cleanup_config($conf);
2739
2740     &$cleanup_config($conf->{pending}, 1);
2741
2742     foreach my $snapname (keys %{$conf->{snapshots}}) {
2743         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2744         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2745     }
2746
2747     # remove 'unusedX' settings if we re-add a volume
2748     foreach my $key (keys %$conf) {
2749         my $value = $conf->{$key};
2750         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2751             delete $conf->{$key};
2752         }
2753     }
2754
2755     my $generate_raw_config = sub {
2756         my ($conf, $pending) = @_;
2757
2758         my $raw = '';
2759
2760         # add description as comment to top of file
2761         if (defined(my $descr = $conf->{description})) {
2762             if ($descr) {
2763                 foreach my $cl (split(/\n/, $descr)) {
2764                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2765                 }
2766             } else {
2767                 $raw .= "#\n" if $pending;
2768             }
2769         }
2770
2771         foreach my $key (sort keys %$conf) {
2772             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2773             $raw .= "$key: $conf->{$key}\n";
2774         }
2775         return $raw;
2776     };
2777
2778     my $raw = &$generate_raw_config($conf);
2779
2780     if (scalar(keys %{$conf->{pending}})){
2781         $raw .= "\n[PENDING]\n";
2782         $raw .= &$generate_raw_config($conf->{pending}, 1);
2783     }
2784
2785     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2786         $raw .= "\n[$snapname]\n";
2787         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2788     }
2789
2790     return $raw;
2791 }
2792
2793 sub load_defaults {
2794
2795     my $res = {};
2796
2797     # we use static defaults from our JSON schema configuration
2798     foreach my $key (keys %$confdesc) {
2799         if (defined(my $default = $confdesc->{$key}->{default})) {
2800             $res->{$key} = $default;
2801         }
2802     }
2803
2804     return $res;
2805 }
2806
2807 sub config_list {
2808     my $vmlist = PVE::Cluster::get_vmlist();
2809     my $res = {};
2810     return $res if !$vmlist || !$vmlist->{ids};
2811     my $ids = $vmlist->{ids};
2812     my $nodename = nodename();
2813
2814     foreach my $vmid (keys %$ids) {
2815         my $d = $ids->{$vmid};
2816         next if !$d->{node} || $d->{node} ne $nodename;
2817         next if !$d->{type} || $d->{type} ne 'qemu';
2818         $res->{$vmid}->{exists} = 1;
2819     }
2820     return $res;
2821 }
2822
2823 # test if VM uses local resources (to prevent migration)
2824 sub check_local_resources {
2825     my ($conf, $noerr) = @_;
2826
2827     my @loc_res = ();
2828
2829     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2830     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2831
2832     push @loc_res, "ivshmem" if $conf->{ivshmem};
2833
2834     foreach my $k (keys %$conf) {
2835         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2836         # sockets are safe: they will recreated be on the target side post-migrate
2837         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2838         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2839     }
2840
2841     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2842
2843     return \@loc_res;
2844 }
2845
2846 # check if used storages are available on all nodes (use by migrate)
2847 sub check_storage_availability {
2848     my ($storecfg, $conf, $node) = @_;
2849
2850     foreach_drive($conf, sub {
2851         my ($ds, $drive) = @_;
2852
2853         my $volid = $drive->{file};
2854         return if !$volid;
2855
2856         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2857         return if !$sid;
2858
2859         # check if storage is available on both nodes
2860         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2861         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2862    });
2863 }
2864
2865 # list nodes where all VM images are available (used by has_feature API)
2866 sub shared_nodes {
2867     my ($conf, $storecfg) = @_;
2868
2869     my $nodelist = PVE::Cluster::get_nodelist();
2870     my $nodehash = { map { $_ => 1 } @$nodelist };
2871     my $nodename = nodename();
2872
2873     foreach_drive($conf, sub {
2874         my ($ds, $drive) = @_;
2875
2876         my $volid = $drive->{file};
2877         return if !$volid;
2878
2879         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2880         if ($storeid) {
2881             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2882             if ($scfg->{disable}) {
2883                 $nodehash = {};
2884             } elsif (my $avail = $scfg->{nodes}) {
2885                 foreach my $node (keys %$nodehash) {
2886                     delete $nodehash->{$node} if !$avail->{$node};
2887                 }
2888             } elsif (!$scfg->{shared}) {
2889                 foreach my $node (keys %$nodehash) {
2890                     delete $nodehash->{$node} if $node ne $nodename
2891                 }
2892             }
2893         }
2894     });
2895
2896     return $nodehash
2897 }
2898
2899 sub check_local_storage_availability {
2900     my ($conf, $storecfg) = @_;
2901
2902     my $nodelist = PVE::Cluster::get_nodelist();
2903     my $nodehash = { map { $_ => {} } @$nodelist };
2904
2905     foreach_drive($conf, sub {
2906         my ($ds, $drive) = @_;
2907
2908         my $volid = $drive->{file};
2909         return if !$volid;
2910
2911         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2912         if ($storeid) {
2913             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2914
2915             if ($scfg->{disable}) {
2916                 foreach my $node (keys %$nodehash) {
2917                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2918                 }
2919             } elsif (my $avail = $scfg->{nodes}) {
2920                 foreach my $node (keys %$nodehash) {
2921                     if (!$avail->{$node}) {
2922                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2923                     }
2924                 }
2925             }
2926         }
2927     });
2928
2929     foreach my $node (values %$nodehash) {
2930         if (my $unavail = $node->{unavailable_storages}) {
2931             $node->{unavailable_storages} = [ sort keys %$unavail ];
2932         }
2933     }
2934
2935     return $nodehash
2936 }
2937
2938 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2939 sub check_running {
2940     my ($vmid, $nocheck, $node) = @_;
2941
2942     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2943     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2944 }
2945
2946 sub vzlist {
2947
2948     my $vzlist = config_list();
2949
2950     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2951
2952     while (defined(my $de = $fd->read)) {
2953         next if $de !~ m/^(\d+)\.pid$/;
2954         my $vmid = $1;
2955         next if !defined($vzlist->{$vmid});
2956         if (my $pid = check_running($vmid)) {
2957             $vzlist->{$vmid}->{pid} = $pid;
2958         }
2959     }
2960
2961     return $vzlist;
2962 }
2963
2964 sub disksize {
2965     my ($storecfg, $conf) = @_;
2966
2967     my $bootdisk = $conf->{bootdisk};
2968     return undef if !$bootdisk;
2969     return undef if !is_valid_drivename($bootdisk);
2970
2971     return undef if !$conf->{$bootdisk};
2972
2973     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2974     return undef if !defined($drive);
2975
2976     return undef if drive_is_cdrom($drive);
2977
2978     my $volid = $drive->{file};
2979     return undef if !$volid;
2980
2981     return $drive->{size};
2982 }
2983
2984 our $vmstatus_return_properties = {
2985     vmid => get_standard_option('pve-vmid'),
2986     status => {
2987         description => "Qemu process status.",
2988         type => 'string',
2989         enum => ['stopped', 'running'],
2990     },
2991     maxmem => {
2992         description => "Maximum memory in bytes.",
2993         type => 'integer',
2994         optional => 1,
2995         renderer => 'bytes',
2996     },
2997     maxdisk => {
2998         description => "Root disk size in bytes.",
2999         type => 'integer',
3000         optional => 1,
3001         renderer => 'bytes',
3002     },
3003     name => {
3004         description => "VM name.",
3005         type => 'string',
3006         optional => 1,
3007     },
3008     qmpstatus => {
3009         description => "Qemu QMP agent status.",
3010         type => 'string',
3011         optional => 1,
3012     },
3013     pid => {
3014         description => "PID of running qemu process.",
3015         type => 'integer',
3016         optional => 1,
3017     },
3018     uptime => {
3019         description => "Uptime.",
3020         type => 'integer',
3021         optional => 1,
3022         renderer => 'duration',
3023     },
3024     cpus => {
3025         description => "Maximum usable CPUs.",
3026         type => 'number',
3027         optional => 1,
3028     },
3029     lock => {
3030         description => "The current config lock, if any.",
3031         type => 'string',
3032         optional => 1,
3033     },
3034     tags => {
3035         description  => "The current configured tags, if any",
3036         type => 'string',
3037         optional => 1,
3038     },
3039 };
3040
3041 my $last_proc_pid_stat;
3042
3043 # get VM status information
3044 # This must be fast and should not block ($full == false)
3045 # We only query KVM using QMP if $full == true (this can be slow)
3046 sub vmstatus {
3047     my ($opt_vmid, $full) = @_;
3048
3049     my $res = {};
3050
3051     my $storecfg = PVE::Storage::config();
3052
3053     my $list = vzlist();
3054     my $defaults = load_defaults();
3055
3056     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
3057
3058     my $cpucount = $cpuinfo->{cpus} || 1;
3059
3060     foreach my $vmid (keys %$list) {
3061         next if $opt_vmid && ($vmid ne $opt_vmid);
3062
3063         my $conf = PVE::QemuConfig->load_config($vmid);
3064
3065         my $d = { vmid => $vmid };
3066         $d->{pid} = $list->{$vmid}->{pid};
3067
3068         # fixme: better status?
3069         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3070
3071         my $size = disksize($storecfg, $conf);
3072         if (defined($size)) {
3073             $d->{disk} = 0; # no info available
3074             $d->{maxdisk} = $size;
3075         } else {
3076             $d->{disk} = 0;
3077             $d->{maxdisk} = 0;
3078         }
3079
3080         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3081             * ($conf->{cores} || $defaults->{cores});
3082         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3083         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3084
3085         $d->{name} = $conf->{name} || "VM $vmid";
3086         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3087             : $defaults->{memory}*(1024*1024);
3088
3089         if ($conf->{balloon}) {
3090             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3091             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3092                 : $defaults->{shares};
3093         }
3094
3095         $d->{uptime} = 0;
3096         $d->{cpu} = 0;
3097         $d->{mem} = 0;
3098
3099         $d->{netout} = 0;
3100         $d->{netin} = 0;
3101
3102         $d->{diskread} = 0;
3103         $d->{diskwrite} = 0;
3104
3105         $d->{template} = PVE::QemuConfig->is_template($conf);
3106
3107         $d->{serial} = 1 if conf_has_serial($conf);
3108         $d->{lock} = $conf->{lock} if $conf->{lock};
3109         $d->{tags} = $conf->{tags} if defined($conf->{tags});
3110
3111         $res->{$vmid} = $d;
3112     }
3113
3114     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3115     foreach my $dev (keys %$netdev) {
3116         next if $dev !~ m/^tap([1-9]\d*)i/;
3117         my $vmid = $1;
3118         my $d = $res->{$vmid};
3119         next if !$d;
3120
3121         $d->{netout} += $netdev->{$dev}->{receive};
3122         $d->{netin} += $netdev->{$dev}->{transmit};
3123
3124         if ($full) {
3125             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3126             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3127         }
3128
3129     }
3130
3131     my $ctime = gettimeofday;
3132
3133     foreach my $vmid (keys %$list) {
3134
3135         my $d = $res->{$vmid};
3136         my $pid = $d->{pid};
3137         next if !$pid;
3138
3139         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3140         next if !$pstat; # not running
3141
3142         my $used = $pstat->{utime} + $pstat->{stime};
3143
3144         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3145
3146         if ($pstat->{vsize}) {
3147             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3148         }
3149
3150         my $old = $last_proc_pid_stat->{$pid};
3151         if (!$old) {
3152             $last_proc_pid_stat->{$pid} = {
3153                 time => $ctime,
3154                 used => $used,
3155                 cpu => 0,
3156             };
3157             next;
3158         }
3159
3160         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3161
3162         if ($dtime > 1000) {
3163             my $dutime = $used -  $old->{used};
3164
3165             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3166             $last_proc_pid_stat->{$pid} = {
3167                 time => $ctime,
3168                 used => $used,
3169                 cpu => $d->{cpu},
3170             };
3171         } else {
3172             $d->{cpu} = $old->{cpu};
3173         }
3174     }
3175
3176     return $res if !$full;
3177
3178     my $qmpclient = PVE::QMPClient->new();
3179
3180     my $ballooncb = sub {
3181         my ($vmid, $resp) = @_;
3182
3183         my $info = $resp->{'return'};
3184         return if !$info->{max_mem};
3185
3186         my $d = $res->{$vmid};
3187
3188         # use memory assigned to VM
3189         $d->{maxmem} = $info->{max_mem};
3190         $d->{balloon} = $info->{actual};
3191
3192         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3193             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3194             $d->{freemem} = $info->{free_mem};
3195         }
3196
3197         $d->{ballooninfo} = $info;
3198     };
3199
3200     my $blockstatscb = sub {
3201         my ($vmid, $resp) = @_;
3202         my $data = $resp->{'return'} || [];
3203         my $totalrdbytes = 0;
3204         my $totalwrbytes = 0;
3205
3206         for my $blockstat (@$data) {
3207             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3208             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3209
3210             $blockstat->{device} =~ s/drive-//;
3211             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3212         }
3213         $res->{$vmid}->{diskread} = $totalrdbytes;
3214         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3215     };
3216
3217     my $statuscb = sub {
3218         my ($vmid, $resp) = @_;
3219
3220         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3221         # this fails if ballon driver is not loaded, so this must be
3222         # the last commnand (following command are aborted if this fails).
3223         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3224
3225         my $status = 'unknown';
3226         if (!defined($status = $resp->{'return'}->{status})) {
3227             warn "unable to get VM status\n";
3228             return;
3229         }
3230
3231         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3232     };
3233
3234     foreach my $vmid (keys %$list) {
3235         next if $opt_vmid && ($vmid ne $opt_vmid);
3236         next if !$res->{$vmid}->{pid}; # not running
3237         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3238     }
3239
3240     $qmpclient->queue_execute(undef, 2);
3241
3242     foreach my $vmid (keys %$list) {
3243         next if $opt_vmid && ($vmid ne $opt_vmid);
3244         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3245     }
3246
3247     return $res;
3248 }
3249
3250 sub foreach_drive {
3251     my ($conf, $func, @param) = @_;
3252
3253     foreach my $ds (valid_drive_names()) {
3254         next if !defined($conf->{$ds});
3255
3256         my $drive = parse_drive($ds, $conf->{$ds});
3257         next if !$drive;
3258
3259         &$func($ds, $drive, @param);
3260     }
3261 }
3262
3263 sub foreach_volid {
3264     my ($conf, $func, @param) = @_;
3265
3266     my $volhash = {};
3267
3268     my $test_volid = sub {
3269         my ($volid, $is_cdrom, $replicate, $shared, $snapname, $size) = @_;
3270
3271         return if !$volid;
3272
3273         $volhash->{$volid}->{cdrom} //= 1;
3274         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3275
3276         $volhash->{$volid}->{replicate} //= 0;
3277         $volhash->{$volid}->{replicate} = 1 if $replicate;
3278
3279         $volhash->{$volid}->{shared} //= 0;
3280         $volhash->{$volid}->{shared} = 1 if $shared;
3281
3282         $volhash->{$volid}->{referenced_in_config} //= 0;
3283         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3284
3285         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3286             if defined($snapname);
3287         $volhash->{$volid}->{size} = $size if $size;
3288     };
3289
3290     foreach_drive($conf, sub {
3291         my ($ds, $drive) = @_;
3292         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef, $drive->{size});
3293     });
3294
3295     foreach my $snapname (keys %{$conf->{snapshots}}) {
3296         my $snap = $conf->{snapshots}->{$snapname};
3297         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3298         foreach_drive($snap, sub {
3299             my ($ds, $drive) = @_;
3300             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3301         });
3302     }
3303
3304     foreach my $volid (keys %$volhash) {
3305         &$func($volid, $volhash->{$volid}, @param);
3306     }
3307 }
3308
3309 sub conf_has_serial {
3310     my ($conf) = @_;
3311
3312     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3313         if ($conf->{"serial$i"}) {
3314             return 1;
3315         }
3316     }
3317
3318     return 0;
3319 }
3320
3321 sub conf_has_audio {
3322     my ($conf, $id) = @_;
3323
3324     $id //= 0;
3325     my $audio = $conf->{"audio$id"};
3326     return undef if !defined($audio);
3327
3328     my $audioproperties = PVE::JSONSchema::parse_property_string($audio_fmt, $audio);
3329     my $audiodriver = $audioproperties->{driver} // 'spice';
3330
3331     return {
3332         dev => $audioproperties->{device},
3333         dev_id => "audiodev$id",
3334         backend => $audiodriver,
3335         backend_id => "$audiodriver-backend${id}",
3336     };
3337 }
3338
3339 sub vga_conf_has_spice {
3340     my ($vga) = @_;
3341
3342     my $vgaconf = parse_vga($vga);
3343     my $vgatype = $vgaconf->{type};
3344     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3345
3346     return $1 || 1;
3347 }
3348
3349 sub is_native($) {
3350     my ($arch) = @_;
3351     return get_host_arch() eq $arch;
3352 }
3353
3354 sub get_vm_arch {
3355     my ($conf) = @_;
3356     return $conf->{arch} // get_host_arch();
3357 }
3358
3359 my $default_machines = {
3360     x86_64 => 'pc',
3361     aarch64 => 'virt',
3362 };
3363
3364 sub get_vm_machine {
3365     my ($conf, $forcemachine, $arch, $add_pve_version) = @_;
3366
3367     my $machine = $forcemachine || $conf->{machine};
3368
3369     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3370         $arch //= 'x86_64';
3371         $machine ||= $default_machines->{$arch};
3372         $machine .= "+pve$PVE::QemuServer::Machine::PVE_MACHINE_VERSION" if $add_pve_version;
3373     }
3374
3375     return $machine;
3376 }
3377
3378 sub get_ovmf_files($) {
3379     my ($arch) = @_;
3380
3381     my $ovmf = $OVMF->{$arch}
3382         or die "no OVMF images known for architecture '$arch'\n";
3383
3384     return @$ovmf;
3385 }
3386
3387 my $Arch2Qemu = {
3388     aarch64 => '/usr/bin/qemu-system-aarch64',
3389     x86_64 => '/usr/bin/qemu-system-x86_64',
3390 };
3391 sub get_command_for_arch($) {
3392     my ($arch) = @_;
3393     return '/usr/bin/kvm' if is_native($arch);
3394
3395     my $cmd = $Arch2Qemu->{$arch}
3396         or die "don't know how to emulate architecture '$arch'\n";
3397     return $cmd;
3398 }
3399
3400 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3401 # to use in a QEMU command line (-cpu element), first array_intersect the result
3402 # of query_supported_ with query_understood_. This is necessary because:
3403 #
3404 # a) query_understood_ returns flags the host cannot use and
3405 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3406 #    flags, but CPU settings - with most of them being flags. Those settings
3407 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3408 #
3409 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3410 # expensive. If you need the value returned from this, you can get it much
3411 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3412 # $accel being 'kvm' or 'tcg'.
3413 #
3414 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3415 # changes, automatically populating pmxcfs.
3416 #
3417 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3418 # since kvm and tcg machines support different flags
3419 #
3420 sub query_supported_cpu_flags {
3421     my ($arch) = @_;
3422
3423     $arch //= get_host_arch();
3424     my $default_machine = $default_machines->{$arch};
3425
3426     my $flags = {};
3427
3428     # FIXME: Once this is merged, the code below should work for ARM as well:
3429     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3430     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3431         $arch eq "aarch64";
3432
3433     my $kvm_supported = defined(kvm_version());
3434     my $qemu_cmd = get_command_for_arch($arch);
3435     my $fakevmid = -1;
3436     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3437
3438     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3439     my $query_supported_run_qemu = sub {
3440         my ($kvm) = @_;
3441
3442         my $flags = {};
3443         my $cmd = [
3444             $qemu_cmd,
3445             '-machine', $default_machine,
3446             '-display', 'none',
3447             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server,nowait",
3448             '-mon', 'chardev=qmp,mode=control',
3449             '-pidfile', $pidfile,
3450             '-S', '-daemonize'
3451         ];
3452
3453         if (!$kvm) {
3454             push @$cmd, '-accel', 'tcg';
3455         }
3456
3457         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3458         die "QEMU flag querying VM exited with code " . $rc if $rc;
3459
3460         eval {
3461             my $cmd_result = mon_cmd(
3462                 $fakevmid,
3463                 'query-cpu-model-expansion',
3464                 type => 'full',
3465                 model => { name => 'host' }
3466             );
3467
3468             my $props = $cmd_result->{model}->{props};
3469             foreach my $prop (keys %$props) {
3470                 next if $props->{$prop} ne '1';
3471                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3472                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3473                 # We only keep those with underscores, to match /proc/cpuinfo
3474                 $prop =~ s/\.|-/_/g;
3475                 $flags->{$prop} = 1;
3476             }
3477         };
3478         my $err = $@;
3479
3480         # force stop with 10 sec timeout and 'nocheck'
3481         # always stop, even if QMP failed
3482         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3483
3484         die $err if $err;
3485
3486         return [ sort keys %$flags ];
3487     };
3488
3489     # We need to query QEMU twice, since KVM and TCG have different supported flags
3490     PVE::QemuConfig->lock_config($fakevmid, sub {
3491         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3492         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3493
3494         if ($kvm_supported) {
3495             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3496             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3497         }
3498     });
3499
3500     return $flags;
3501 }
3502
3503 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3504 my $understood_cpu_flag_dir = "/usr/share/kvm";
3505 sub query_understood_cpu_flags {
3506     my $arch = get_host_arch();
3507     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3508
3509     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3510         if ! -e $filepath;
3511
3512     my $raw = file_get_contents($filepath);
3513     $raw =~ s/^\s+|\s+$//g;
3514     my @flags = split(/\s+/, $raw);
3515
3516     return \@flags;
3517 }
3518
3519 sub get_cpu_options {
3520     my ($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough) = @_;
3521
3522     my $cpuFlags = [];
3523     my $ostype = $conf->{ostype};
3524
3525     my $cpu = $kvm ? "kvm64" : "qemu64";
3526     if ($arch eq 'aarch64') {
3527         $cpu = 'cortex-a57';
3528     }
3529     my $hv_vendor_id;
3530     if (my $cputype = $conf->{cpu}) {
3531         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3532             or die "Cannot parse cpu description: $cputype\n";
3533         $cpu = $cpuconf->{cputype};
3534         $kvm_off = 1 if $cpuconf->{hidden};
3535         $hv_vendor_id = $cpuconf->{'hv-vendor-id'};
3536
3537         if (defined(my $flags = $cpuconf->{flags})) {
3538             push @$cpuFlags, split(";", $flags);
3539         }
3540     }
3541
3542     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3543
3544     push @$cpuFlags , '-x2apic' if $ostype && $ostype eq 'solaris';
3545
3546     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3547
3548     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3549
3550     if (min_version($machine_version, 2, 3) && $arch eq 'x86_64') {
3551
3552         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3553         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3554     }
3555
3556     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_version, $conf->{bios}, $gpu_passthrough, $hv_vendor_id) if $kvm;
3557
3558     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3559
3560     push @$cpuFlags, 'kvm=off' if $kvm_off;
3561
3562     if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3563         push @$cpuFlags, "vendor=${cpu_vendor}"
3564             if $cpu_vendor ne 'default';
3565     } elsif ($arch ne 'aarch64') {
3566         die "internal error"; # should not happen
3567     }
3568
3569     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3570
3571     return ('-cpu', $cpu);
3572 }
3573
3574 sub config_to_command {
3575     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3576
3577     my $cmd = [];
3578     my $globalFlags = [];
3579     my $machineFlags = [];
3580     my $rtcFlags = [];
3581     my $devices = [];
3582     my $pciaddr = '';
3583     my $bridges = {};
3584     my $vernum = 0; # unknown
3585     my $ostype = $conf->{ostype};
3586     my $winversion = windows_version($ostype);
3587     my $kvm = $conf->{kvm};
3588     my $nodename = nodename();
3589
3590     my $arch = get_vm_arch($conf);
3591     my $kvm_binary = get_command_for_arch($arch);
3592     my $kvmver = kvm_user_version($kvm_binary);
3593
3594     my $add_pve_version = min_version($kvmver, 4, 1);
3595
3596     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3597     my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type, $kvmver);
3598     $kvm //= 1 if is_native($arch);
3599
3600     $machine_version =~ m/(\d+)\.(\d+)/;
3601     die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type', please upgrade node '$nodename'\n"
3602         if !PVE::QemuServer::min_version($kvmver, $1, $2);
3603
3604     if ($kvm) {
3605         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3606             if !defined kvm_version();
3607     }
3608
3609     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3610         $vernum = $1*1000000+$2*1000;
3611     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3612         $vernum = $1*1000000+$2*1000+$3;
3613     }
3614
3615     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3616
3617     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3618     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3619     my $use_old_bios_files = undef;
3620     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3621
3622     my $cpuunits = defined($conf->{cpuunits}) ?
3623             $conf->{cpuunits} : $defaults->{cpuunits};
3624
3625     push @$cmd, $kvm_binary;
3626
3627     push @$cmd, '-id', $vmid;
3628
3629     my $vmname = $conf->{name} || "vm$vmid";
3630
3631     push @$cmd, '-name', $vmname;
3632
3633     my $use_virtio = 0;
3634
3635     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3636     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3637     push @$cmd, '-mon', "chardev=qmp,mode=control";
3638
3639     if (min_version($machine_version, 2, 12)) {
3640         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3641         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3642     }
3643
3644     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3645
3646     push @$cmd, '-daemonize';
3647
3648     if ($conf->{smbios1}) {
3649         my $smbios_conf = parse_smbios1($conf->{smbios1});
3650         if ($smbios_conf->{base64}) {
3651             # Do not pass base64 flag to qemu
3652             delete $smbios_conf->{base64};
3653             my $smbios_string = "";
3654             foreach my $key (keys %$smbios_conf) {
3655                 my $value;
3656                 if ($key eq "uuid") {
3657                     $value = $smbios_conf->{uuid}
3658                 } else {
3659                     $value = decode_base64($smbios_conf->{$key});
3660                 }
3661                 # qemu accepts any binary data, only commas need escaping by double comma
3662                 $value =~ s/,/,,/g;
3663                 $smbios_string .= "," . $key . "=" . $value if $value;
3664             }
3665             push @$cmd, '-smbios', "type=1" . $smbios_string;
3666         } else {
3667             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3668         }
3669     }
3670
3671     if ($conf->{vmgenid}) {
3672         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3673     }
3674
3675     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3676     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3677         die "uefi base image not found\n" if ! -f $ovmf_code;
3678
3679         my $path;
3680         my $format;
3681         if (my $efidisk = $conf->{efidisk0}) {
3682             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3683             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3684             $format = $d->{format};
3685             if ($storeid) {
3686                 $path = PVE::Storage::path($storecfg, $d->{file});
3687                 if (!defined($format)) {
3688                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3689                     $format = qemu_img_format($scfg, $volname);
3690                 }
3691             } else {
3692                 $path = $d->{file};
3693                 die "efidisk format must be specified\n"
3694                     if !defined($format);
3695             }
3696         } else {
3697             warn "no efidisk configured! Using temporary efivars disk.\n";
3698             $path = "/tmp/$vmid-ovmf.fd";
3699             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3700             $format = 'raw';
3701         }
3702
3703         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3704         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3705     }
3706
3707     # load q35 config
3708     if ($q35) {
3709         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3710         if (min_version($machine_version, 4, 0)) {
3711             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3712         } else {
3713             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3714         }
3715     }
3716
3717     # add usb controllers
3718     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3719     push @$devices, @usbcontrollers if @usbcontrollers;
3720     my $vga = parse_vga($conf->{vga});
3721
3722     my $qxlnum = vga_conf_has_spice($conf->{vga});
3723     $vga->{type} = 'qxl' if $qxlnum;
3724
3725     if (!$vga->{type}) {
3726         if ($arch eq 'aarch64') {
3727             $vga->{type} = 'virtio';
3728         } elsif (min_version($machine_version, 2, 9)) {
3729             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3730         } else {
3731             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3732         }
3733     }
3734
3735     # enable absolute mouse coordinates (needed by vnc)
3736     my $tablet;
3737     if (defined($conf->{tablet})) {
3738         $tablet = $conf->{tablet};
3739     } else {
3740         $tablet = $defaults->{tablet};
3741         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3742         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3743     }
3744
3745     if ($tablet) {
3746         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3747         my $kbd = print_keyboarddevice_full($conf, $arch);
3748         push @$devices, '-device', $kbd if defined($kbd);
3749     }
3750
3751     my $kvm_off = 0;
3752     my $gpu_passthrough;
3753
3754     # host pci devices
3755     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3756         my $id = "hostpci$i";
3757         my $d = parse_hostpci($conf->{$id});
3758         next if !$d;
3759
3760         if (my $pcie = $d->{pcie}) {
3761             die "q35 machine model is not enabled" if !$q35;
3762             # win7 wants to have the pcie devices directly on the pcie bus
3763             # instead of in the root port
3764             if ($winversion == 7) {
3765                 $pciaddr = print_pcie_addr("${id}bus0");
3766             } else {
3767                 # add more root ports if needed, 4 are present by default
3768                 # by pve-q35 cfgs, rest added here on demand.
3769                 if ($i > 3) {
3770                     push @$devices, '-device', print_pcie_root_port($i);
3771                 }
3772                 $pciaddr = print_pcie_addr($id);
3773             }
3774         } else {
3775             $pciaddr = print_pci_addr($id, $bridges, $arch, $machine_type);
3776         }
3777
3778         my $xvga = '';
3779         if ($d->{'x-vga'}) {
3780             $xvga = ',x-vga=on' if !($conf->{bios} && $conf->{bios} eq 'ovmf');
3781             $kvm_off = 1;
3782             $vga->{type} = 'none' if !defined($conf->{vga});
3783             $gpu_passthrough = 1;
3784         }
3785
3786         my $pcidevices = $d->{pciid};
3787         my $multifunction = 1 if @$pcidevices > 1;
3788
3789         my $sysfspath;
3790         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3791             my $pci_id = $pcidevices->[0]->{id};
3792             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3793             $sysfspath = "/sys/bus/pci/devices/$pci_id/$uuid";
3794         } elsif ($d->{mdev}) {
3795             warn "ignoring mediated device '$id' with multifunction device\n";
3796         }
3797
3798         my $j=0;
3799         foreach my $pcidevice (@$pcidevices) {
3800             my $devicestr = "vfio-pci";
3801
3802             if ($sysfspath) {
3803                 $devicestr .= ",sysfsdev=$sysfspath";
3804             } else {
3805                 $devicestr .= ",host=$pcidevice->{id}";
3806             }
3807
3808             my $mf_addr = $multifunction ? ".$j" : '';
3809             $devicestr .= ",id=${id}${mf_addr}${pciaddr}${mf_addr}";
3810
3811             if ($j == 0) {
3812                 $devicestr .= ',rombar=0' if defined($d->{rombar}) && !$d->{rombar};
3813                 $devicestr .= "$xvga";
3814                 $devicestr .= ",multifunction=on" if $multifunction;
3815                 $devicestr .= ",romfile=/usr/share/kvm/$d->{romfile}" if $d->{romfile};
3816             }
3817
3818             push @$devices, '-device', $devicestr;
3819             $j++;
3820         }
3821     }
3822
3823     # usb devices
3824     my $usb_dev_features = {};
3825     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3826
3827     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features);
3828     push @$devices, @usbdevices if @usbdevices;
3829     # serial devices
3830     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3831         if (my $path = $conf->{"serial$i"}) {
3832             if ($path eq 'socket') {
3833                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3834                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3835                 # On aarch64, serial0 is the UART device. Qemu only allows
3836                 # connecting UART devices via the '-serial' command line, as
3837                 # the device has a fixed slot on the hardware...
3838                 if ($arch eq 'aarch64' && $i == 0) {
3839                     push @$devices, '-serial', "chardev:serial$i";
3840                 } else {
3841                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3842                 }
3843             } else {
3844                 die "no such serial device\n" if ! -c $path;
3845                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3846                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3847             }
3848         }
3849     }
3850
3851     # parallel devices
3852     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3853         if (my $path = $conf->{"parallel$i"}) {
3854             die "no such parallel device\n" if ! -c $path;
3855             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3856             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3857             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3858         }
3859     }
3860
3861     if (my $audio = conf_has_audio($conf)) {
3862
3863         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3864
3865         my $id = $audio->{dev_id};
3866         if ($audio->{dev} eq 'AC97') {
3867             push @$devices, '-device', "AC97,id=${id}${audiopciaddr}";
3868         } elsif ($audio->{dev} =~ /intel\-hda$/) {
3869             push @$devices, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3870             push @$devices, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0";
3871             push @$devices, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1";
3872         } else {
3873             die "unkown audio device '$audio->{dev}', implement me!";
3874         }
3875
3876         push @$devices, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3877     }
3878
3879     my $sockets = 1;
3880     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3881     $sockets = $conf->{sockets} if  $conf->{sockets};
3882
3883     my $cores = $conf->{cores} || 1;
3884
3885     my $maxcpus = $sockets * $cores;
3886
3887     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3888
3889     my $allowed_vcpus = $cpuinfo->{cpus};
3890
3891     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3892         if ($allowed_vcpus < $maxcpus);
3893
3894     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3895
3896         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3897         for (my $i = 2; $i <= $vcpus; $i++)  {
3898             my $cpustr = print_cpu_device($conf,$i);
3899             push @$cmd, '-device', $cpustr;
3900         }
3901
3902     } else {
3903
3904         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3905     }
3906     push @$cmd, '-nodefaults';
3907
3908     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3909
3910     my $bootindex_hash = {};
3911     my $i = 1;
3912     foreach my $o (split(//, $bootorder)) {
3913         $bootindex_hash->{$o} = $i*100;
3914         $i++;
3915     }
3916
3917     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3918
3919     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3920
3921     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3922
3923     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3924         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3925         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3926         push @$cmd,  '-vnc', "unix:$socket,password";
3927     } else {
3928         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3929         push @$cmd, '-nographic';
3930     }
3931
3932     # time drift fix
3933     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3934
3935     my $useLocaltime = $conf->{localtime};
3936
3937     if ($winversion >= 5) { # windows
3938         $useLocaltime = 1 if !defined($conf->{localtime});
3939
3940         # use time drift fix when acpi is enabled
3941         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3942             $tdf = 1 if !defined($conf->{tdf});
3943         }
3944     }
3945
3946     if ($winversion >= 6) {
3947         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3948         push @$cmd, '-no-hpet';
3949     }
3950
3951     push @$rtcFlags, 'driftfix=slew' if $tdf;
3952
3953     if (!$kvm) {
3954         push @$machineFlags, 'accel=tcg';
3955     }
3956
3957     if ($machine_type) {
3958         push @$machineFlags, "type=${machine_type}";
3959     }
3960
3961     if (($conf->{startdate}) && ($conf->{startdate} ne 'now')) {
3962         push @$rtcFlags, "base=$conf->{startdate}";
3963     } elsif ($useLocaltime) {
3964         push @$rtcFlags, 'base=localtime';
3965     }
3966
3967     push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3968
3969     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3970
3971     push @$cmd, '-S' if $conf->{freeze};
3972
3973     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3974
3975     my $guest_agent = parse_guest_agent($conf);
3976
3977     if ($guest_agent->{enabled}) {
3978         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3979         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3980
3981         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3982             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3983             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3984             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3985         } elsif ($guest_agent->{type} eq 'isa') {
3986             push @$devices, '-device', "isa-serial,chardev=qga0";
3987         }
3988     }
3989
3990     my $spice_port;
3991
3992     if ($qxlnum) {
3993         if ($qxlnum > 1) {
3994             if ($winversion){
3995                 for(my $i = 1; $i < $qxlnum; $i++){
3996                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3997                 }
3998             } else {
3999                 # assume other OS works like Linux
4000                 my ($ram, $vram) = ("134217728", "67108864");
4001                 if ($vga->{memory}) {
4002                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
4003                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
4004                 }
4005                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
4006                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
4007             }
4008         }
4009
4010         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
4011
4012         my $pfamily = PVE::Tools::get_host_address_family($nodename);
4013         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
4014         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
4015
4016         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
4017         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
4018         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
4019
4020         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
4021         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
4022
4023         my $spice_enhancement = PVE::JSONSchema::parse_property_string($spice_enhancements_fmt, $conf->{spice_enhancements} // '');
4024         if ($spice_enhancement->{foldersharing}) {
4025             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
4026             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
4027         }
4028
4029         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
4030         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming};
4031         push @$devices, '-spice', "$spice_opts";
4032     }
4033
4034     # enable balloon by default, unless explicitly disabled
4035     if (!defined($conf->{balloon}) || $conf->{balloon}) {
4036         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
4037         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
4038     }
4039
4040     if ($conf->{watchdog}) {
4041         my $wdopts = parse_watchdog($conf->{watchdog});
4042         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
4043         my $watchdog = $wdopts->{model} || 'i6300esb';
4044         push @$devices, '-device', "$watchdog$pciaddr";
4045         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
4046     }
4047
4048     my $vollist = [];
4049     my $scsicontroller = {};
4050     my $ahcicontroller = {};
4051     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
4052
4053     # Add iscsi initiator name if available
4054     if (my $initiator = get_initiator_name()) {
4055         push @$devices, '-iscsi', "initiator-name=$initiator";
4056     }
4057
4058     foreach_drive($conf, sub {
4059         my ($ds, $drive) = @_;
4060
4061         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
4062             push @$vollist, $drive->{file};
4063         }
4064
4065         # ignore efidisk here, already added in bios/fw handling code above
4066         return if $drive->{interface} eq 'efidisk';
4067
4068         $use_virtio = 1 if $ds =~ m/^virtio/;
4069
4070         if (drive_is_cdrom ($drive)) {
4071             if ($bootindex_hash->{d}) {
4072                 $drive->{bootindex} = $bootindex_hash->{d};
4073                 $bootindex_hash->{d} += 1;
4074             }
4075         } else {
4076             if ($bootindex_hash->{c}) {
4077                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
4078                 $bootindex_hash->{c} += 1;
4079             }
4080         }
4081
4082         if($drive->{interface} eq 'virtio'){
4083            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
4084         }
4085
4086         if ($drive->{interface} eq 'scsi') {
4087
4088             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
4089
4090             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
4091             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
4092
4093             my $iothread = '';
4094             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
4095                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
4096                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
4097             } elsif ($drive->{iothread}) {
4098                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
4099             }
4100
4101             my $queues = '';
4102             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
4103                 $queues = ",num_queues=$drive->{queues}";
4104             }
4105
4106             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
4107             $scsicontroller->{$controller}=1;
4108         }
4109
4110         if ($drive->{interface} eq 'sata') {
4111            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
4112            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
4113            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
4114            $ahcicontroller->{$controller}=1;
4115         }
4116
4117         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
4118         push @$devices, '-drive',$drive_cmd;
4119         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4120     });
4121
4122     for (my $i = 0; $i < $MAX_NETS; $i++) {
4123          next if !$conf->{"net$i"};
4124          my $d = parse_net($conf->{"net$i"});
4125          next if !$d;
4126
4127          $use_virtio = 1 if $d->{model} eq 'virtio';
4128
4129          if ($bootindex_hash->{n}) {
4130             $d->{bootindex} = $bootindex_hash->{n};
4131             $bootindex_hash->{n} += 1;
4132          }
4133
4134          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
4135          push @$devices, '-netdev', $netdevfull;
4136
4137          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
4138          push @$devices, '-device', $netdevicefull;
4139     }
4140
4141     if ($conf->{ivshmem}) {
4142         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4143
4144         my $bus;
4145         if ($q35) {
4146             $bus = print_pcie_addr("ivshmem");
4147         } else {
4148             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4149         }
4150
4151         my $ivshmem_name = $ivshmem->{name} // $vmid;
4152         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4153
4154         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4155         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
4156     }
4157
4158     if (!$q35) {
4159         # add pci bridges
4160         if (min_version($machine_version, 2, 3)) {
4161            $bridges->{1} = 1;
4162            $bridges->{2} = 1;
4163         }
4164
4165         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4166
4167         for my $k (sort {$b cmp $a} keys %$bridges) {
4168             $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
4169             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
4170         }
4171     }
4172
4173     push @$cmd, @$devices;
4174     push @$cmd, '-rtc', join(',', @$rtcFlags)
4175         if scalar(@$rtcFlags);
4176     push @$cmd, '-machine', join(',', @$machineFlags)
4177         if scalar(@$machineFlags);
4178     push @$cmd, '-global', join(',', @$globalFlags)
4179         if scalar(@$globalFlags);
4180
4181     if (my $vmstate = $conf->{vmstate}) {
4182         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4183         push @$vollist, $vmstate;
4184         push @$cmd, '-loadstate', $statepath;
4185         print "activating and using '$vmstate' as vmstate\n";
4186     }
4187
4188     # add custom args
4189     if ($conf->{args}) {
4190         my $aa = PVE::Tools::split_args($conf->{args});
4191         push @$cmd, @$aa;
4192     }
4193
4194     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4195 }
4196
4197 sub spice_port {
4198     my ($vmid) = @_;
4199
4200     my $res = mon_cmd($vmid, 'query-spice');
4201
4202     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4203 }
4204
4205 sub vm_devices_list {
4206     my ($vmid) = @_;
4207
4208     my $res = mon_cmd($vmid, 'query-pci');
4209     my $devices_to_check = [];
4210     my $devices = {};
4211     foreach my $pcibus (@$res) {
4212         push @$devices_to_check, @{$pcibus->{devices}},
4213     }
4214
4215     while (@$devices_to_check) {
4216         my $to_check = [];
4217         for my $d (@$devices_to_check) {
4218             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4219             next if !$d->{'pci_bridge'};
4220
4221             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4222             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4223         }
4224         $devices_to_check = $to_check;
4225     }
4226
4227     my $resblock = mon_cmd($vmid, 'query-block');
4228     foreach my $block (@$resblock) {
4229         if($block->{device} =~ m/^drive-(\S+)/){
4230                 $devices->{$1} = 1;
4231         }
4232     }
4233
4234     my $resmice = mon_cmd($vmid, 'query-mice');
4235     foreach my $mice (@$resmice) {
4236         if ($mice->{name} eq 'QEMU HID Tablet') {
4237             $devices->{tablet} = 1;
4238             last;
4239         }
4240     }
4241
4242     # for usb devices there is no query-usb
4243     # but we can iterate over the entries in
4244     # qom-list path=/machine/peripheral
4245     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4246     foreach my $per (@$resperipheral) {
4247         if ($per->{name} =~ m/^usb\d+$/) {
4248             $devices->{$per->{name}} = 1;
4249         }
4250     }
4251
4252     return $devices;
4253 }
4254
4255 sub vm_deviceplug {
4256     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4257
4258     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4259
4260     my $devices_list = vm_devices_list($vmid);
4261     return 1 if defined($devices_list->{$deviceid});
4262
4263     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
4264
4265     if ($deviceid eq 'tablet') {
4266
4267         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4268
4269     } elsif ($deviceid eq 'keyboard') {
4270
4271         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4272
4273     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4274
4275         die "usb hotplug currently not reliable\n";
4276         # since we can't reliably hot unplug all added usb devices
4277         # and usb passthrough disables live migration
4278         # we disable usb hotplugging for now
4279         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4280
4281     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4282
4283         qemu_iothread_add($vmid, $deviceid, $device);
4284
4285         qemu_driveadd($storecfg, $vmid, $device);
4286         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4287
4288         qemu_deviceadd($vmid, $devicefull);
4289         eval { qemu_deviceaddverify($vmid, $deviceid); };
4290         if (my $err = $@) {
4291             eval { qemu_drivedel($vmid, $deviceid); };
4292             warn $@ if $@;
4293             die $err;
4294         }
4295
4296     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4297
4298
4299         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4300         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4301         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4302
4303         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4304
4305         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4306             qemu_iothread_add($vmid, $deviceid, $device);
4307             $devicefull .= ",iothread=iothread-$deviceid";
4308         }
4309
4310         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4311             $devicefull .= ",num_queues=$device->{queues}";
4312         }
4313
4314         qemu_deviceadd($vmid, $devicefull);
4315         qemu_deviceaddverify($vmid, $deviceid);
4316
4317     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4318
4319         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4320         qemu_driveadd($storecfg, $vmid, $device);
4321
4322         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4323         eval { qemu_deviceadd($vmid, $devicefull); };
4324         if (my $err = $@) {
4325             eval { qemu_drivedel($vmid, $deviceid); };
4326             warn $@ if $@;
4327             die $err;
4328         }
4329
4330     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4331
4332         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4333
4334         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4335         my $use_old_bios_files = undef;
4336         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4337
4338         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4339         qemu_deviceadd($vmid, $netdevicefull);
4340         eval {
4341             qemu_deviceaddverify($vmid, $deviceid);
4342             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4343         };
4344         if (my $err = $@) {
4345             eval { qemu_netdevdel($vmid, $deviceid); };
4346             warn $@ if $@;
4347             die $err;
4348         }
4349
4350     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4351
4352         my $bridgeid = $2;
4353         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4354         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4355
4356         qemu_deviceadd($vmid, $devicefull);
4357         qemu_deviceaddverify($vmid, $deviceid);
4358
4359     } else {
4360         die "can't hotplug device '$deviceid'\n";
4361     }
4362
4363     return 1;
4364 }
4365
4366 # fixme: this should raise exceptions on error!
4367 sub vm_deviceunplug {
4368     my ($vmid, $conf, $deviceid) = @_;
4369
4370     my $devices_list = vm_devices_list($vmid);
4371     return 1 if !defined($devices_list->{$deviceid});
4372
4373     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4374
4375     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4376
4377         qemu_devicedel($vmid, $deviceid);
4378
4379     } elsif ($deviceid =~ m/^usb\d+$/) {
4380
4381         die "usb hotplug currently not reliable\n";
4382         # when unplugging usb devices this way,
4383         # there may be remaining usb controllers/hubs
4384         # so we disable it for now
4385         qemu_devicedel($vmid, $deviceid);
4386         qemu_devicedelverify($vmid, $deviceid);
4387
4388     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4389
4390         qemu_devicedel($vmid, $deviceid);
4391         qemu_devicedelverify($vmid, $deviceid);
4392         qemu_drivedel($vmid, $deviceid);
4393         qemu_iothread_del($conf, $vmid, $deviceid);
4394
4395     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4396
4397         qemu_devicedel($vmid, $deviceid);
4398         qemu_devicedelverify($vmid, $deviceid);
4399         qemu_iothread_del($conf, $vmid, $deviceid);
4400
4401     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4402
4403         qemu_devicedel($vmid, $deviceid);
4404         qemu_drivedel($vmid, $deviceid);
4405         qemu_deletescsihw($conf, $vmid, $deviceid);
4406
4407     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4408
4409         qemu_devicedel($vmid, $deviceid);
4410         qemu_devicedelverify($vmid, $deviceid);
4411         qemu_netdevdel($vmid, $deviceid);
4412
4413     } else {
4414         die "can't unplug device '$deviceid'\n";
4415     }
4416
4417     return 1;
4418 }
4419
4420 sub qemu_deviceadd {
4421     my ($vmid, $devicefull) = @_;
4422
4423     $devicefull = "driver=".$devicefull;
4424     my %options =  split(/[=,]/, $devicefull);
4425
4426     mon_cmd($vmid, "device_add" , %options);
4427 }
4428
4429 sub qemu_devicedel {
4430     my ($vmid, $deviceid) = @_;
4431
4432     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4433 }
4434
4435 sub qemu_iothread_add {
4436     my($vmid, $deviceid, $device) = @_;
4437
4438     if ($device->{iothread}) {
4439         my $iothreads = vm_iothreads_list($vmid);
4440         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4441     }
4442 }
4443
4444 sub qemu_iothread_del {
4445     my($conf, $vmid, $deviceid) = @_;
4446
4447     my $confid = $deviceid;
4448     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4449         $confid = 'scsi' . $1;
4450     }
4451     my $device = parse_drive($confid, $conf->{$confid});
4452     if ($device->{iothread}) {
4453         my $iothreads = vm_iothreads_list($vmid);
4454         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4455     }
4456 }
4457
4458 sub qemu_objectadd {
4459     my($vmid, $objectid, $qomtype) = @_;
4460
4461     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4462
4463     return 1;
4464 }
4465
4466 sub qemu_objectdel {
4467     my($vmid, $objectid) = @_;
4468
4469     mon_cmd($vmid, "object-del", id => $objectid);
4470
4471     return 1;
4472 }
4473
4474 sub qemu_driveadd {
4475     my ($storecfg, $vmid, $device) = @_;
4476
4477     my $drive = print_drive_full($storecfg, $vmid, $device);
4478     $drive =~ s/\\/\\\\/g;
4479     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4480
4481     # If the command succeeds qemu prints: "OK"
4482     return 1 if $ret =~ m/OK/s;
4483
4484     die "adding drive failed: $ret\n";
4485 }
4486
4487 sub qemu_drivedel {
4488     my($vmid, $deviceid) = @_;
4489
4490     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4491     $ret =~ s/^\s+//;
4492
4493     return 1 if $ret eq "";
4494
4495     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4496     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4497
4498     die "deleting drive $deviceid failed : $ret\n";
4499 }
4500
4501 sub qemu_deviceaddverify {
4502     my ($vmid, $deviceid) = @_;
4503
4504     for (my $i = 0; $i <= 5; $i++) {
4505          my $devices_list = vm_devices_list($vmid);
4506          return 1 if defined($devices_list->{$deviceid});
4507          sleep 1;
4508     }
4509
4510     die "error on hotplug device '$deviceid'\n";
4511 }
4512
4513
4514 sub qemu_devicedelverify {
4515     my ($vmid, $deviceid) = @_;
4516
4517     # need to verify that the device is correctly removed as device_del
4518     # is async and empty return is not reliable
4519
4520     for (my $i = 0; $i <= 5; $i++) {
4521          my $devices_list = vm_devices_list($vmid);
4522          return 1 if !defined($devices_list->{$deviceid});
4523          sleep 1;
4524     }
4525
4526     die "error on hot-unplugging device '$deviceid'\n";
4527 }
4528
4529 sub qemu_findorcreatescsihw {
4530     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4531
4532     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4533
4534     my $scsihwid="$controller_prefix$controller";
4535     my $devices_list = vm_devices_list($vmid);
4536
4537     if(!defined($devices_list->{$scsihwid})) {
4538         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4539     }
4540
4541     return 1;
4542 }
4543
4544 sub qemu_deletescsihw {
4545     my ($conf, $vmid, $opt) = @_;
4546
4547     my $device = parse_drive($opt, $conf->{$opt});
4548
4549     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4550         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4551         return 1;
4552     }
4553
4554     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4555
4556     my $devices_list = vm_devices_list($vmid);
4557     foreach my $opt (keys %{$devices_list}) {
4558         if (PVE::QemuServer::is_valid_drivename($opt)) {
4559             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4560             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4561                 return 1;
4562             }
4563         }
4564     }
4565
4566     my $scsihwid="scsihw$controller";
4567
4568     vm_deviceunplug($vmid, $conf, $scsihwid);
4569
4570     return 1;
4571 }
4572
4573 sub qemu_add_pci_bridge {
4574     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4575
4576     my $bridges = {};
4577
4578     my $bridgeid;
4579
4580     print_pci_addr($device, $bridges, $arch, $machine_type);
4581
4582     while (my ($k, $v) = each %$bridges) {
4583         $bridgeid = $k;
4584     }
4585     return 1 if !defined($bridgeid) || $bridgeid < 1;
4586
4587     my $bridge = "pci.$bridgeid";
4588     my $devices_list = vm_devices_list($vmid);
4589
4590     if (!defined($devices_list->{$bridge})) {
4591         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4592     }
4593
4594     return 1;
4595 }
4596
4597 sub qemu_set_link_status {
4598     my ($vmid, $device, $up) = @_;
4599
4600     mon_cmd($vmid, "set_link", name => $device,
4601                up => $up ? JSON::true : JSON::false);
4602 }
4603
4604 sub qemu_netdevadd {
4605     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4606
4607     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4608     my %options =  split(/[=,]/, $netdev);
4609
4610     mon_cmd($vmid, "netdev_add",  %options);
4611     return 1;
4612 }
4613
4614 sub qemu_netdevdel {
4615     my ($vmid, $deviceid) = @_;
4616
4617     mon_cmd($vmid, "netdev_del", id => $deviceid);
4618 }
4619
4620 sub qemu_usb_hotplug {
4621     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4622
4623     return if !$device;
4624
4625     # remove the old one first
4626     vm_deviceunplug($vmid, $conf, $deviceid);
4627
4628     # check if xhci controller is necessary and available
4629     if ($device->{usb3}) {
4630
4631         my $devicelist = vm_devices_list($vmid);
4632
4633         if (!$devicelist->{xhci}) {
4634             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4635             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4636         }
4637     }
4638     my $d = parse_usb_device($device->{host});
4639     $d->{usb3} = $device->{usb3};
4640
4641     # add the new one
4642     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4643 }
4644
4645 sub qemu_cpu_hotplug {
4646     my ($vmid, $conf, $vcpus) = @_;
4647
4648     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4649
4650     my $sockets = 1;
4651     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4652     $sockets = $conf->{sockets} if  $conf->{sockets};
4653     my $cores = $conf->{cores} || 1;
4654     my $maxcpus = $sockets * $cores;
4655
4656     $vcpus = $maxcpus if !$vcpus;
4657
4658     die "you can't add more vcpus than maxcpus\n"
4659         if $vcpus > $maxcpus;
4660
4661     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4662
4663     if ($vcpus < $currentvcpus) {
4664
4665         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4666
4667             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4668                 qemu_devicedel($vmid, "cpu$i");
4669                 my $retry = 0;
4670                 my $currentrunningvcpus = undef;
4671                 while (1) {
4672                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus");
4673                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4674                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4675                     $retry++;
4676                     sleep 1;
4677                 }
4678                 #update conf after each succesfull cpu unplug
4679                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4680                 PVE::QemuConfig->write_config($vmid, $conf);
4681             }
4682         } else {
4683             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4684         }
4685
4686         return;
4687     }
4688
4689     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus");
4690     die "vcpus in running vm does not match its configuration\n"
4691         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4692
4693     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4694
4695         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4696             my $cpustr = print_cpu_device($conf, $i);
4697             qemu_deviceadd($vmid, $cpustr);
4698
4699             my $retry = 0;
4700             my $currentrunningvcpus = undef;
4701             while (1) {
4702                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus");
4703                 last if scalar(@{$currentrunningvcpus}) == $i;
4704                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4705                 sleep 1;
4706                 $retry++;
4707             }
4708             #update conf after each succesfull cpu hotplug
4709             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4710             PVE::QemuConfig->write_config($vmid, $conf);
4711         }
4712     } else {
4713
4714         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4715             mon_cmd($vmid, "cpu-add", id => int($i));
4716         }
4717     }
4718 }
4719
4720 sub qemu_block_set_io_throttle {
4721     my ($vmid, $deviceid,
4722         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4723         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4724         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4725         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4726
4727     return if !check_running($vmid) ;
4728
4729     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4730         bps => int($bps),
4731         bps_rd => int($bps_rd),
4732         bps_wr => int($bps_wr),
4733         iops => int($iops),
4734         iops_rd => int($iops_rd),
4735         iops_wr => int($iops_wr),
4736         bps_max => int($bps_max),
4737         bps_rd_max => int($bps_rd_max),
4738         bps_wr_max => int($bps_wr_max),
4739         iops_max => int($iops_max),
4740         iops_rd_max => int($iops_rd_max),
4741         iops_wr_max => int($iops_wr_max),
4742         bps_max_length => int($bps_max_length),
4743         bps_rd_max_length => int($bps_rd_max_length),
4744         bps_wr_max_length => int($bps_wr_max_length),
4745         iops_max_length => int($iops_max_length),
4746         iops_rd_max_length => int($iops_rd_max_length),
4747         iops_wr_max_length => int($iops_wr_max_length),
4748     );
4749
4750 }
4751
4752 # old code, only used to shutdown old VM after update
4753 sub __read_avail {
4754     my ($fh, $timeout) = @_;
4755
4756     my $sel = new IO::Select;
4757     $sel->add($fh);
4758
4759     my $res = '';
4760     my $buf;
4761
4762     my @ready;
4763     while (scalar (@ready = $sel->can_read($timeout))) {
4764         my $count;
4765         if ($count = $fh->sysread($buf, 8192)) {
4766             if ($buf =~ /^(.*)\(qemu\) $/s) {
4767                 $res .= $1;
4768                 last;
4769             } else {
4770                 $res .= $buf;
4771             }
4772         } else {
4773             if (!defined($count)) {
4774                 die "$!\n";
4775             }
4776             last;
4777         }
4778     }
4779
4780     die "monitor read timeout\n" if !scalar(@ready);
4781
4782     return $res;
4783 }
4784
4785 sub qemu_block_resize {
4786     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4787
4788     my $running = check_running($vmid);
4789
4790     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4791
4792     return if !$running;
4793
4794     mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4795
4796 }
4797
4798 sub qemu_volume_snapshot {
4799     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4800
4801     my $running = check_running($vmid);
4802
4803     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4804         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4805     } else {
4806         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4807     }
4808 }
4809
4810 sub qemu_volume_snapshot_delete {
4811     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4812
4813     my $running = check_running($vmid);
4814
4815     if($running) {
4816
4817         $running = undef;
4818         my $conf = PVE::QemuConfig->load_config($vmid);
4819         foreach_drive($conf, sub {
4820             my ($ds, $drive) = @_;
4821             $running = 1 if $drive->{file} eq $volid;
4822         });
4823     }
4824
4825     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4826         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4827     } else {
4828         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4829     }
4830 }
4831
4832 sub set_migration_caps {
4833     my ($vmid) = @_;
4834
4835     my $cap_ref = [];
4836
4837     my $enabled_cap = {
4838         "auto-converge" => 1,
4839         "xbzrle" => 1,
4840         "x-rdma-pin-all" => 0,
4841         "zero-blocks" => 0,
4842         "compress" => 0
4843     };
4844
4845     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4846
4847     for my $supported_capability (@$supported_capabilities) {
4848         push @$cap_ref, {
4849             capability => $supported_capability->{capability},
4850             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4851         };
4852     }
4853
4854     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4855 }
4856
4857 my $fast_plug_option = {
4858     'lock' => 1,
4859     'name' => 1,
4860     'onboot' => 1,
4861     'shares' => 1,
4862     'startup' => 1,
4863     'description' => 1,
4864     'protection' => 1,
4865     'vmstatestorage' => 1,
4866     'hookscript' => 1,
4867     'tags' => 1,
4868 };
4869
4870 # hotplug changes in [PENDING]
4871 # $selection hash can be used to only apply specified options, for
4872 # example: { cores => 1 } (only apply changed 'cores')
4873 # $errors ref is used to return error messages
4874 sub vmconfig_hotplug_pending {
4875     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4876
4877     my $defaults = load_defaults();
4878     my $arch = get_vm_arch($conf);
4879     my $machine_type = get_vm_machine($conf, undef, $arch);
4880
4881     # commit values which do not have any impact on running VM first
4882     # Note: those option cannot raise errors, we we do not care about
4883     # $selection and always apply them.
4884
4885     my $add_error = sub {
4886         my ($opt, $msg) = @_;
4887         $errors->{$opt} = "hotplug problem - $msg";
4888     };
4889
4890     my $changes = 0;
4891     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4892         if ($fast_plug_option->{$opt}) {
4893             $conf->{$opt} = $conf->{pending}->{$opt};
4894             delete $conf->{pending}->{$opt};
4895             $changes = 1;
4896         }
4897     }
4898
4899     if ($changes) {
4900         PVE::QemuConfig->write_config($vmid, $conf);
4901     }
4902
4903     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4904
4905     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4906     foreach my $opt (sort keys %$pending_delete_hash) {
4907         next if $selection && !$selection->{$opt};
4908         my $force = $pending_delete_hash->{$opt}->{force};
4909         eval {
4910             if ($opt eq 'hotplug') {
4911                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4912             } elsif ($opt eq 'tablet') {
4913                 die "skip\n" if !$hotplug_features->{usb};
4914                 if ($defaults->{tablet}) {
4915                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4916                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4917                         if $arch eq 'aarch64';
4918                 } else {
4919                     vm_deviceunplug($vmid, $conf, 'tablet');
4920                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4921                 }
4922             } elsif ($opt =~ m/^usb\d+/) {
4923                 die "skip\n";
4924                 # since we cannot reliably hot unplug usb devices
4925                 # we are disabling it
4926                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4927                 vm_deviceunplug($vmid, $conf, $opt);
4928             } elsif ($opt eq 'vcpus') {
4929                 die "skip\n" if !$hotplug_features->{cpu};
4930                 qemu_cpu_hotplug($vmid, $conf, undef);
4931             } elsif ($opt eq 'balloon') {
4932                 # enable balloon device is not hotpluggable
4933                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4934                 # here we reset the ballooning value to memory
4935                 my $balloon = $conf->{memory} || $defaults->{memory};
4936                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4937             } elsif ($fast_plug_option->{$opt}) {
4938                 # do nothing
4939             } elsif ($opt =~ m/^net(\d+)$/) {
4940                 die "skip\n" if !$hotplug_features->{network};
4941                 vm_deviceunplug($vmid, $conf, $opt);
4942             } elsif (is_valid_drivename($opt)) {
4943                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4944                 vm_deviceunplug($vmid, $conf, $opt);
4945                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4946             } elsif ($opt =~ m/^memory$/) {
4947                 die "skip\n" if !$hotplug_features->{memory};
4948                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4949             } elsif ($opt eq 'cpuunits') {
4950                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4951             } elsif ($opt eq 'cpulimit') {
4952                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4953             } else {
4954                 die "skip\n";
4955             }
4956         };
4957         if (my $err = $@) {
4958             &$add_error($opt, $err) if $err ne "skip\n";
4959         } else {
4960             delete $conf->{$opt};
4961             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4962         }
4963     }
4964
4965     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4966     $apply_pending_cloudinit = sub {
4967         return if $apply_pending_cloudinit_done; # once is enough
4968         $apply_pending_cloudinit_done = 1; # once is enough
4969
4970         my ($key, $value) = @_;
4971
4972         my @cloudinit_opts = keys %$confdesc_cloudinit;
4973         foreach my $opt (keys %{$conf->{pending}}) {
4974             next if !grep { $_ eq $opt } @cloudinit_opts;
4975             $conf->{$opt} = delete $conf->{pending}->{$opt};
4976         }
4977
4978         my $new_conf = { %$conf };
4979         $new_conf->{$key} = $value;
4980         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4981     };
4982
4983     foreach my $opt (keys %{$conf->{pending}}) {
4984         next if $selection && !$selection->{$opt};
4985         my $value = $conf->{pending}->{$opt};
4986         eval {
4987             if ($opt eq 'hotplug') {
4988                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4989             } elsif ($opt eq 'tablet') {
4990                 die "skip\n" if !$hotplug_features->{usb};
4991                 if ($value == 1) {
4992                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4993                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4994                         if $arch eq 'aarch64';
4995                 } elsif ($value == 0) {
4996                     vm_deviceunplug($vmid, $conf, 'tablet');
4997                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4998                 }
4999             } elsif ($opt =~ m/^usb\d+$/) {
5000                 die "skip\n";
5001                 # since we cannot reliably hot unplug usb devices
5002                 # we are disabling it
5003                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
5004                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
5005                 die "skip\n" if !$d;
5006                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
5007             } elsif ($opt eq 'vcpus') {
5008                 die "skip\n" if !$hotplug_features->{cpu};
5009                 qemu_cpu_hotplug($vmid, $conf, $value);
5010             } elsif ($opt eq 'balloon') {
5011                 # enable/disable balloning device is not hotpluggable
5012                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
5013                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
5014                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
5015
5016                 # allow manual ballooning if shares is set to zero
5017                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
5018                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
5019                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5020                 }
5021             } elsif ($opt =~ m/^net(\d+)$/) {
5022                 # some changes can be done without hotplug
5023                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
5024                                     $vmid, $opt, $value, $arch, $machine_type);
5025             } elsif (is_valid_drivename($opt)) {
5026                 die "skip\n" if $opt eq 'efidisk0';
5027                 # some changes can be done without hotplug
5028                 my $drive = parse_drive($opt, $value);
5029                 if (drive_is_cloudinit($drive)) {
5030                     &$apply_pending_cloudinit($opt, $value);
5031                 }
5032                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5033                                      $vmid, $opt, $value, 1, $arch, $machine_type);
5034             } elsif ($opt =~ m/^memory$/) { #dimms
5035                 die "skip\n" if !$hotplug_features->{memory};
5036                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
5037             } elsif ($opt eq 'cpuunits') {
5038                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
5039             } elsif ($opt eq 'cpulimit') {
5040                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
5041                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
5042             } else {
5043                 die "skip\n";  # skip non-hot-pluggable options
5044             }
5045         };
5046         if (my $err = $@) {
5047             &$add_error($opt, $err) if $err ne "skip\n";
5048         } else {
5049             $conf->{$opt} = $value;
5050             delete $conf->{pending}->{$opt};
5051         }
5052     }
5053
5054     PVE::QemuConfig->write_config($vmid, $conf);
5055 }
5056
5057 sub try_deallocate_drive {
5058     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5059
5060     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5061         my $volid = $drive->{file};
5062         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5063             my $sid = PVE::Storage::parse_volume_id($volid);
5064             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5065
5066             # check if the disk is really unused
5067             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5068                 if is_volume_in_use($storecfg, $conf, $key, $volid);
5069             PVE::Storage::vdisk_free($storecfg, $volid);
5070             return 1;
5071         } else {
5072             # If vm is not owner of this disk remove from config
5073             return 1;
5074         }
5075     }
5076
5077     return undef;
5078 }
5079
5080 sub vmconfig_delete_or_detach_drive {
5081     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5082
5083     my $drive = parse_drive($opt, $conf->{$opt});
5084
5085     my $rpcenv = PVE::RPCEnvironment::get();
5086     my $authuser = $rpcenv->get_user();
5087
5088     if ($force) {
5089         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5090         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5091     } else {
5092         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5093     }
5094 }
5095
5096
5097
5098 sub vmconfig_apply_pending {
5099     my ($vmid, $conf, $storecfg, $errors) = @_;
5100
5101     my $add_apply_error = sub {
5102         my ($opt, $msg) = @_;
5103         my $err_msg = "unable to apply pending change $opt : $msg";
5104         $errors->{$opt} = $err_msg;
5105         warn $err_msg;
5106     };
5107
5108     # cold plug
5109
5110     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5111     foreach my $opt (sort keys %$pending_delete_hash) {
5112         my $force = $pending_delete_hash->{$opt}->{force};
5113         eval {
5114             if ($opt =~ m/^unused/) {
5115                 die "internal error";
5116             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5117                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5118             }
5119         };
5120         if (my $err = $@) {
5121             $add_apply_error->($opt, $err);
5122         } else {
5123             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5124             delete $conf->{$opt};
5125         }
5126     }
5127
5128     PVE::QemuConfig->cleanup_pending($conf);
5129
5130     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5131         next if $opt eq 'delete'; # just to be sure
5132         eval {
5133             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5134                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5135             }
5136         };
5137         if (my $err = $@) {
5138             $add_apply_error->($opt, $err);
5139         } else {
5140             $conf->{$opt} = delete $conf->{pending}->{$opt};
5141         }
5142     }
5143
5144     # write all changes at once to avoid unnecessary i/o
5145     PVE::QemuConfig->write_config($vmid, $conf);
5146 }
5147
5148 my $safe_num_ne = sub {
5149     my ($a, $b) = @_;
5150
5151     return 0 if !defined($a) && !defined($b);
5152     return 1 if !defined($a);
5153     return 1 if !defined($b);
5154
5155     return $a != $b;
5156 };
5157
5158 my $safe_string_ne = sub {
5159     my ($a, $b) = @_;
5160
5161     return 0 if !defined($a) && !defined($b);
5162     return 1 if !defined($a);
5163     return 1 if !defined($b);
5164
5165     return $a ne $b;
5166 };
5167
5168 sub vmconfig_update_net {
5169     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5170
5171     my $newnet = parse_net($value);
5172
5173     if ($conf->{$opt}) {
5174         my $oldnet = parse_net($conf->{$opt});
5175
5176         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5177             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5178             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5179             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5180
5181             # for non online change, we try to hot-unplug
5182             die "skip\n" if !$hotplug;
5183             vm_deviceunplug($vmid, $conf, $opt);
5184         } else {
5185
5186             die "internal error" if $opt !~ m/net(\d+)/;
5187             my $iface = "tap${vmid}i$1";
5188
5189             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5190                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5191                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5192                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5193                 PVE::Network::tap_unplug($iface);
5194                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5195             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5196                 # Rate can be applied on its own but any change above needs to
5197                 # include the rate in tap_plug since OVS resets everything.
5198                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5199             }
5200
5201             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5202                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5203             }
5204
5205             return 1;
5206         }
5207     }
5208
5209     if ($hotplug) {
5210         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5211     } else {
5212         die "skip\n";
5213     }
5214 }
5215
5216 sub vmconfig_update_disk {
5217     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
5218
5219     # fixme: do we need force?
5220
5221     my $drive = parse_drive($opt, $value);
5222
5223     if ($conf->{$opt}) {
5224
5225         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
5226
5227             my $media = $drive->{media} || 'disk';
5228             my $oldmedia = $old_drive->{media} || 'disk';
5229             die "unable to change media type\n" if $media ne $oldmedia;
5230
5231             if (!drive_is_cdrom($old_drive)) {
5232
5233                 if ($drive->{file} ne $old_drive->{file}) {
5234
5235                     die "skip\n" if !$hotplug;
5236
5237                     # unplug and register as unused
5238                     vm_deviceunplug($vmid, $conf, $opt);
5239                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5240
5241                 } else {
5242                     # update existing disk
5243
5244                     # skip non hotpluggable value
5245                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5246                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5247                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5248                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
5249                         die "skip\n";
5250                     }
5251
5252                     # apply throttle
5253                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5254                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5255                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5256                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5257                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5258                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5259                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5260                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5261                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5262                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5263                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5264                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5265                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5266                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5267                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5268                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5269                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5270                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5271
5272                         qemu_block_set_io_throttle($vmid,"drive-$opt",
5273                                                    ($drive->{mbps} || 0)*1024*1024,
5274                                                    ($drive->{mbps_rd} || 0)*1024*1024,
5275                                                    ($drive->{mbps_wr} || 0)*1024*1024,
5276                                                    $drive->{iops} || 0,
5277                                                    $drive->{iops_rd} || 0,
5278                                                    $drive->{iops_wr} || 0,
5279                                                    ($drive->{mbps_max} || 0)*1024*1024,
5280                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
5281                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
5282                                                    $drive->{iops_max} || 0,
5283                                                    $drive->{iops_rd_max} || 0,
5284                                                    $drive->{iops_wr_max} || 0,
5285                                                    $drive->{bps_max_length} || 1,
5286                                                    $drive->{bps_rd_max_length} || 1,
5287                                                    $drive->{bps_wr_max_length} || 1,
5288                                                    $drive->{iops_max_length} || 1,
5289                                                    $drive->{iops_rd_max_length} || 1,
5290                                                    $drive->{iops_wr_max_length} || 1);
5291
5292                     }
5293
5294                     return 1;
5295                 }
5296
5297             } else { # cdrom
5298
5299                 if ($drive->{file} eq 'none') {
5300                     mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5301                     if (drive_is_cloudinit($old_drive)) {
5302                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5303                     }
5304                 } else {
5305                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5306                     mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5307                     mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5308                 }
5309
5310                 return 1;
5311             }
5312         }
5313     }
5314
5315     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5316     # hotplug new disks
5317     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5318     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5319 }
5320
5321 sub vm_start {
5322     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5323         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage, $timeout) = @_;
5324
5325     PVE::QemuConfig->lock_config($vmid, sub {
5326         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5327
5328         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5329
5330         my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended');
5331
5332         PVE::QemuConfig->check_lock($conf)
5333             if !($skiplock || $is_suspended);
5334
5335         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5336
5337         # clean up leftover reboot request files
5338         eval { clear_reboot_request($vmid); };
5339         warn $@ if $@;
5340
5341         if (!$statefile && scalar(keys %{$conf->{pending}})) {
5342             vmconfig_apply_pending($vmid, $conf, $storecfg);
5343             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5344         }
5345
5346         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5347
5348         my $defaults = load_defaults();
5349
5350         # set environment variable useful inside network script
5351         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5352
5353         my $local_volumes = {};
5354
5355         if ($targetstorage) {
5356             foreach_drive($conf, sub {
5357                 my ($ds, $drive) = @_;
5358
5359                 return if drive_is_cdrom($drive);
5360
5361                 my $volid = $drive->{file};
5362
5363                 return if !$volid;
5364
5365                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5366
5367                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5368                 return if $scfg->{shared};
5369                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5370             });
5371
5372             my $format = undef;
5373
5374             foreach my $opt (sort keys %$local_volumes) {
5375
5376                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5377                 my $drive = parse_drive($opt, $conf->{$opt});
5378
5379                 # If a remote storage is specified and the format of the original
5380                 # volume is not available there, fall back to the default format.
5381                 # Otherwise use the same format as the original.
5382                 if ($targetstorage && $targetstorage ne "1") {
5383                     $storeid = $targetstorage;
5384                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5385                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5386                     my $fileFormat = qemu_img_format($scfg, $volname);
5387                     $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
5388                 } else {
5389                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5390                     $format = qemu_img_format($scfg, $volid);
5391                 }
5392
5393                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5394                 my $newdrive = $drive;
5395                 $newdrive->{format} = $format;
5396                 $newdrive->{file} = $newvolid;
5397                 my $drivestr = print_drive($newdrive);
5398                 $local_volumes->{$opt} = $drivestr;
5399                 #pass drive to conf for command line
5400                 $conf->{$opt} = $drivestr;
5401             }
5402         }
5403
5404         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5405
5406         if ($is_suspended) {
5407             # enforce machine type on suspended vm to ensure HW compatibility
5408             $forcemachine = $conf->{runningmachine};
5409             print "Resuming suspended VM\n";
5410         }
5411
5412         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5413
5414         my $migration_ip;
5415         my $get_migration_ip = sub {
5416             my ($cidr, $nodename) = @_;
5417
5418             return $migration_ip if defined($migration_ip);
5419
5420             if (!defined($cidr)) {
5421                 my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5422                 $cidr = $dc_conf->{migration}->{network};
5423             }
5424
5425             if (defined($cidr)) {
5426                 my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5427
5428                 die "could not get IP: no address configured on local " .
5429                     "node for network '$cidr'\n" if scalar(@$ips) == 0;
5430
5431                 die "could not get IP: multiple addresses configured on local " .
5432                     "node for network '$cidr'\n" if scalar(@$ips) > 1;
5433
5434                 $migration_ip = @$ips[0];
5435             }
5436
5437             $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5438                 if !defined($migration_ip);
5439
5440             return $migration_ip;
5441         };
5442
5443         my $migrate_uri;
5444         if ($statefile) {
5445             if ($statefile eq 'tcp') {
5446                 my $localip = "localhost";
5447                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5448                 my $nodename = nodename();
5449
5450                 if (!defined($migration_type)) {
5451                     if (defined($datacenterconf->{migration}->{type})) {
5452                         $migration_type = $datacenterconf->{migration}->{type};
5453                     } else {
5454                         $migration_type = 'secure';
5455                     }
5456                 }
5457
5458                 if ($migration_type eq 'insecure') {
5459                     $localip = $get_migration_ip->($migration_network, $nodename);
5460                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5461                 }
5462
5463                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5464                 my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5465                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5466                 push @$cmd, '-incoming', $migrate_uri;
5467                 push @$cmd, '-S';
5468
5469             } elsif ($statefile eq 'unix') {
5470                 # should be default for secure migrations as a ssh TCP forward
5471                 # tunnel is not deterministic reliable ready and fails regurarly
5472                 # to set up in time, so use UNIX socket forwards
5473                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5474                 unlink $socket_addr;
5475
5476                 $migrate_uri = "unix:$socket_addr";
5477
5478                 push @$cmd, '-incoming', $migrate_uri;
5479                 push @$cmd, '-S';
5480
5481             } elsif (-e $statefile) {
5482                 push @$cmd, '-loadstate', $statefile;
5483             } else {
5484                 my $statepath = PVE::Storage::path($storecfg, $statefile);
5485                 push @$vollist, $statefile;
5486                 push @$cmd, '-loadstate', $statepath;
5487             }
5488         } elsif ($paused) {
5489             push @$cmd, '-S';
5490         }
5491
5492         # host pci devices
5493         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5494           my $d = parse_hostpci($conf->{"hostpci$i"});
5495           next if !$d;
5496           my $pcidevices = $d->{pciid};
5497           foreach my $pcidevice (@$pcidevices) {
5498                 my $pciid = $pcidevice->{id};
5499
5500                 my $info = PVE::SysFSTools::pci_device_info("$pciid");
5501                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5502                 die "no pci device info for device '$pciid'\n" if !$info;
5503
5504                 if ($d->{mdev}) {
5505                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5506                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5507                 } else {
5508                     die "can't unbind/bind pci group to vfio '$pciid'\n"
5509                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5510                     die "can't reset pci device '$pciid'\n"
5511                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5512                 }
5513           }
5514         }
5515
5516         PVE::Storage::activate_volumes($storecfg, $vollist);
5517
5518         eval {
5519             run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5520                 outfunc => sub {}, errfunc => sub {});
5521         };
5522         # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5523         # timeout should be more than enough here...
5524         PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5525
5526         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5527                                                   : $defaults->{cpuunits};
5528
5529         my $start_timeout = $timeout // config_aware_timeout($conf, $is_suspended);
5530         my %run_params = (
5531             timeout => $statefile ? undef : $start_timeout,
5532             umask => 0077,
5533             noerr => 1,
5534         );
5535
5536         # when migrating, prefix QEMU output so other side can pick up any
5537         # errors that might occur and show the user
5538         if ($migratedfrom) {
5539             $run_params{quiet} = 1;
5540             $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5541         }
5542
5543         my %properties = (
5544             Slice => 'qemu.slice',
5545             KillMode => 'none',
5546             CPUShares => $cpuunits
5547         );
5548
5549         if (my $cpulimit = $conf->{cpulimit}) {
5550             $properties{CPUQuota} = int($cpulimit * 100);
5551         }
5552         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5553
5554         my $run_qemu = sub {
5555             PVE::Tools::run_fork sub {
5556                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5557
5558                 my $exitcode = run_command($cmd, %run_params);
5559                 die "QEMU exited with code $exitcode\n" if $exitcode;
5560             };
5561         };
5562
5563         if ($conf->{hugepages}) {
5564
5565             my $code = sub {
5566                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5567                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5568
5569                 PVE::QemuServer::Memory::hugepages_mount();
5570                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5571
5572                 eval { $run_qemu->() };
5573                 if (my $err = $@) {
5574                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5575                     die $err;
5576                 }
5577
5578                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5579             };
5580             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5581
5582         } else {
5583             eval { $run_qemu->() };
5584         }
5585
5586         if (my $err = $@) {
5587             # deactivate volumes if start fails
5588             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5589             die "start failed: $err";
5590         }
5591
5592         print "migration listens on $migrate_uri\n" if $migrate_uri;
5593
5594         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5595             eval { mon_cmd($vmid, "cont"); };
5596             warn $@ if $@;
5597         }
5598
5599         #start nbd server for storage migration
5600         if ($targetstorage) {
5601             my $nodename = nodename();
5602             my $localip = $get_migration_ip->($migration_network, $nodename);
5603             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5604             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5605
5606             mon_cmd($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${storage_migrate_port}" } } );
5607
5608             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5609
5610             foreach my $opt (sort keys %$local_volumes) {
5611                 my $volid = $local_volumes->{$opt};
5612                 mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5613                 my $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}:exportname=drive-$opt";
5614                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5615             }
5616         }
5617
5618         if ($migratedfrom) {
5619             eval {
5620                 set_migration_caps($vmid);
5621             };
5622             warn $@ if $@;
5623
5624             if ($spice_port) {
5625                 print "spice listens on port $spice_port\n";
5626                 if ($spice_ticket) {
5627                     mon_cmd($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5628                     mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5629                 }
5630             }
5631
5632         } else {
5633             mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5634                 if !$statefile && $conf->{balloon};
5635
5636             foreach my $opt (keys %$conf) {
5637                 next if $opt !~  m/^net\d+$/;
5638                 my $nicconf = parse_net($conf->{$opt});
5639                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5640             }
5641         }
5642
5643         mon_cmd($vmid, 'qom-set',
5644                     path => "machine/peripheral/balloon0",
5645                     property => "guest-stats-polling-interval",
5646                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5647
5648         if ($is_suspended) {
5649             print "Resumed VM, removing state\n";
5650             if (my $vmstate = $conf->{vmstate}) {
5651                 PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5652                 PVE::Storage::vdisk_free($storecfg, $vmstate);
5653             }
5654             delete $conf->@{qw(lock vmstate runningmachine)};
5655             PVE::QemuConfig->write_config($vmid, $conf);
5656         }
5657
5658         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5659     });
5660 }
5661
5662 sub vm_commandline {
5663     my ($storecfg, $vmid, $snapname) = @_;
5664
5665     my $conf = PVE::QemuConfig->load_config($vmid);
5666     my $forcemachine;
5667
5668     if ($snapname) {
5669         my $snapshot = $conf->{snapshots}->{$snapname};
5670         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5671
5672         # check for a 'runningmachine' in snapshot
5673         $forcemachine = $snapshot->{runningmachine} if $snapshot->{runningmachine};
5674
5675         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5676
5677         $conf = $snapshot;
5678     }
5679
5680     my $defaults = load_defaults();
5681
5682     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5683
5684     return PVE::Tools::cmd2string($cmd);
5685 }
5686
5687 sub vm_reset {
5688     my ($vmid, $skiplock) = @_;
5689
5690     PVE::QemuConfig->lock_config($vmid, sub {
5691
5692         my $conf = PVE::QemuConfig->load_config($vmid);
5693
5694         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5695
5696         mon_cmd($vmid, "system_reset");
5697     });
5698 }
5699
5700 sub get_vm_volumes {
5701     my ($conf) = @_;
5702
5703     my $vollist = [];
5704     foreach_volid($conf, sub {
5705         my ($volid, $attr) = @_;
5706
5707         return if $volid =~ m|^/|;
5708
5709         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5710         return if !$sid;
5711
5712         push @$vollist, $volid;
5713     });
5714
5715     return $vollist;
5716 }
5717
5718 sub vm_stop_cleanup {
5719     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5720
5721     eval {
5722
5723         if (!$keepActive) {
5724             my $vollist = get_vm_volumes($conf);
5725             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5726         }
5727
5728         foreach my $ext (qw(mon qmp pid vnc qga)) {
5729             unlink "/var/run/qemu-server/${vmid}.$ext";
5730         }
5731
5732         if ($conf->{ivshmem}) {
5733             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5734             # just delete it for now, VMs which have this already open do not
5735             # are affected, but new VMs will get a separated one. If this
5736             # becomes an issue we either add some sort of ref-counting or just
5737             # add a "don't delete on stop" flag to the ivshmem format.
5738             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5739         }
5740
5741         foreach my $key (keys %$conf) {
5742             next if $key !~ m/^hostpci(\d+)$/;
5743             my $hostpciindex = $1;
5744             my $d = parse_hostpci($conf->{$key});
5745             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5746
5747             foreach my $pci (@{$d->{pciid}}) {
5748                 my $pciid = $pci->{id};
5749                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5750             }
5751         }
5752
5753         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5754     };
5755     warn $@ if $@; # avoid errors - just warn
5756 }
5757
5758 # call only in locked context
5759 sub _do_vm_stop {
5760     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5761
5762     my $pid = check_running($vmid, $nocheck);
5763     return if !$pid;
5764
5765     my $conf;
5766     if (!$nocheck) {
5767         $conf = PVE::QemuConfig->load_config($vmid);
5768         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5769         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5770             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5771             $timeout = $opts->{down} if $opts->{down};
5772         }
5773         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5774     }
5775
5776     eval {
5777         if ($shutdown) {
5778             if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5779                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5780             } else {
5781                 mon_cmd($vmid, "system_powerdown");
5782             }
5783         } else {
5784             mon_cmd($vmid, "quit");
5785         }
5786     };
5787     my $err = $@;
5788
5789     if (!$err) {
5790         $timeout = 60 if !defined($timeout);
5791
5792         my $count = 0;
5793         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5794             $count++;
5795             sleep 1;
5796         }
5797
5798         if ($count >= $timeout) {
5799             if ($force) {
5800                 warn "VM still running - terminating now with SIGTERM\n";
5801                 kill 15, $pid;
5802             } else {
5803                 die "VM quit/powerdown failed - got timeout\n";
5804             }
5805         } else {
5806             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5807             return;
5808         }
5809     } else {
5810         if ($force) {
5811             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5812             kill 15, $pid;
5813         } else {
5814             die "VM quit/powerdown failed\n";
5815         }
5816     }
5817
5818     # wait again
5819     $timeout = 10;
5820
5821     my $count = 0;
5822     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5823         $count++;
5824         sleep 1;
5825     }
5826
5827     if ($count >= $timeout) {
5828         warn "VM still running - terminating now with SIGKILL\n";
5829         kill 9, $pid;
5830         sleep 1;
5831     }
5832
5833     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5834 }
5835
5836 # Note: use $nocheck to skip tests if VM configuration file exists.
5837 # We need that when migration VMs to other nodes (files already moved)
5838 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5839 sub vm_stop {
5840     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5841
5842     $force = 1 if !defined($force) && !$shutdown;
5843
5844     if ($migratedfrom){
5845         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5846         kill 15, $pid if $pid;
5847         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5848         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5849         return;
5850     }
5851
5852     PVE::QemuConfig->lock_config($vmid, sub {
5853         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5854    });
5855 }
5856
5857 sub vm_reboot {
5858     my ($vmid, $timeout) = @_;
5859
5860     PVE::QemuConfig->lock_config($vmid, sub {
5861         eval {
5862
5863             # only reboot if running, as qmeventd starts it again on a stop event
5864             return if !check_running($vmid);
5865
5866             create_reboot_request($vmid);
5867
5868             my $storecfg = PVE::Storage::config();
5869             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5870
5871         };
5872         if (my $err = $@) {
5873             # avoid that the next normal shutdown will be confused for a reboot
5874             clear_reboot_request($vmid);
5875             die $err;
5876         }
5877    });
5878 }
5879
5880 # note: if using the statestorage parameter, the caller has to check privileges
5881 sub vm_suspend {
5882     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5883
5884     my $conf;
5885     my $path;
5886     my $storecfg;
5887     my $vmstate;
5888
5889     PVE::QemuConfig->lock_config($vmid, sub {
5890
5891         $conf = PVE::QemuConfig->load_config($vmid);
5892
5893         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5894         PVE::QemuConfig->check_lock($conf)
5895             if !($skiplock || $is_backing_up);
5896
5897         die "cannot suspend to disk during backup\n"
5898             if $is_backing_up && $includestate;
5899
5900         if ($includestate) {
5901             $conf->{lock} = 'suspending';
5902             my $date = strftime("%Y-%m-%d", localtime(time()));
5903             $storecfg = PVE::Storage::config();
5904             if (!$statestorage) {
5905                 $statestorage = find_vmstate_storage($conf, $storecfg);
5906                 # check permissions for the storage
5907                 my $rpcenv = PVE::RPCEnvironment::get();
5908                 if ($rpcenv->{type} ne 'cli') {
5909                     my $authuser = $rpcenv->get_user();
5910                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5911                 }
5912             }
5913
5914
5915             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5916             $path = PVE::Storage::path($storecfg, $vmstate);
5917             PVE::QemuConfig->write_config($vmid, $conf);
5918         } else {
5919             mon_cmd($vmid, "stop");
5920         }
5921     });
5922
5923     if ($includestate) {
5924         # save vm state
5925         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5926
5927         eval {
5928             mon_cmd($vmid, "savevm-start", statefile => $path);
5929             for(;;) {
5930                 my $state = mon_cmd($vmid, "query-savevm");
5931                 if (!$state->{status}) {
5932                     die "savevm not active\n";
5933                 } elsif ($state->{status} eq 'active') {
5934                     sleep(1);
5935                     next;
5936                 } elsif ($state->{status} eq 'completed') {
5937                     print "State saved, quitting\n";
5938                     last;
5939                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5940                     die "query-savevm failed with error '$state->{error}'\n"
5941                 } else {
5942                     die "query-savevm returned status '$state->{status}'\n";
5943                 }
5944             }
5945         };
5946         my $err = $@;
5947
5948         PVE::QemuConfig->lock_config($vmid, sub {
5949             $conf = PVE::QemuConfig->load_config($vmid);
5950             if ($err) {
5951                 # cleanup, but leave suspending lock, to indicate something went wrong
5952                 eval {
5953                     mon_cmd($vmid, "savevm-end");
5954                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5955                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5956                     delete $conf->@{qw(vmstate runningmachine)};
5957                     PVE::QemuConfig->write_config($vmid, $conf);
5958                 };
5959                 warn $@ if $@;
5960                 die $err;
5961             }
5962
5963             die "lock changed unexpectedly\n"
5964                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5965
5966             mon_cmd($vmid, "quit");
5967             $conf->{lock} = 'suspended';
5968             PVE::QemuConfig->write_config($vmid, $conf);
5969         });
5970     }
5971 }
5972
5973 sub vm_resume {
5974     my ($vmid, $skiplock, $nocheck) = @_;
5975
5976     PVE::QemuConfig->lock_config($vmid, sub {
5977         my $res = mon_cmd($vmid, 'query-status');
5978         my $resume_cmd = 'cont';
5979
5980         if ($res->{status} && $res->{status} eq 'suspended') {
5981             $resume_cmd = 'system_wakeup';
5982         }
5983
5984         if (!$nocheck) {
5985
5986             my $conf = PVE::QemuConfig->load_config($vmid);
5987
5988             PVE::QemuConfig->check_lock($conf)
5989                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5990         }
5991
5992         mon_cmd($vmid, $resume_cmd);
5993     });
5994 }
5995
5996 sub vm_sendkey {
5997     my ($vmid, $skiplock, $key) = @_;
5998
5999     PVE::QemuConfig->lock_config($vmid, sub {
6000
6001         my $conf = PVE::QemuConfig->load_config($vmid);
6002
6003         # there is no qmp command, so we use the human monitor command
6004         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
6005         die $res if $res ne '';
6006     });
6007 }
6008
6009 # vzdump restore implementaion
6010
6011 sub tar_archive_read_firstfile {
6012     my $archive = shift;
6013
6014     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6015
6016     # try to detect archive type first
6017     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6018         die "unable to open file '$archive'\n";
6019     my $firstfile = <$fh>;
6020     kill 15, $pid;
6021     close $fh;
6022
6023     die "ERROR: archive contaions no data\n" if !$firstfile;
6024     chomp $firstfile;
6025
6026     return $firstfile;
6027 }
6028
6029 sub tar_restore_cleanup {
6030     my ($storecfg, $statfile) = @_;
6031
6032     print STDERR "starting cleanup\n";
6033
6034     if (my $fd = IO::File->new($statfile, "r")) {
6035         while (defined(my $line = <$fd>)) {
6036             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6037                 my $volid = $2;
6038                 eval {
6039                     if ($volid =~ m|^/|) {
6040                         unlink $volid || die 'unlink failed\n';
6041                     } else {
6042                         PVE::Storage::vdisk_free($storecfg, $volid);
6043                     }
6044                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6045                 };
6046                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6047             } else {
6048                 print STDERR "unable to parse line in statfile - $line";
6049             }
6050         }
6051         $fd->close();
6052     }
6053 }
6054
6055 sub restore_archive {
6056     my ($archive, $vmid, $user, $opts) = @_;
6057
6058     my $format = $opts->{format};
6059     my $comp;
6060
6061     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
6062         $format = 'tar' if !$format;
6063         $comp = 'gzip';
6064     } elsif ($archive =~ m/\.tar$/) {
6065         $format = 'tar' if !$format;
6066     } elsif ($archive =~ m/.tar.lzo$/) {
6067         $format = 'tar' if !$format;
6068         $comp = 'lzop';
6069     } elsif ($archive =~ m/\.vma$/) {
6070         $format = 'vma' if !$format;
6071     } elsif ($archive =~ m/\.vma\.gz$/) {
6072         $format = 'vma' if !$format;
6073         $comp = 'gzip';
6074     } elsif ($archive =~ m/\.vma\.lzo$/) {
6075         $format = 'vma' if !$format;
6076         $comp = 'lzop';
6077     } else {
6078         $format = 'vma' if !$format; # default
6079     }
6080
6081     # try to detect archive format
6082     if ($format eq 'tar') {
6083         return restore_tar_archive($archive, $vmid, $user, $opts);
6084     } else {
6085         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6086     }
6087 }
6088
6089 sub restore_update_config_line {
6090     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
6091
6092     return if $line =~ m/^\#qmdump\#/;
6093     return if $line =~ m/^\#vzdump\#/;
6094     return if $line =~ m/^lock:/;
6095     return if $line =~ m/^unused\d+:/;
6096     return if $line =~ m/^parent:/;
6097
6098     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6099     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6100         # try to convert old 1.X settings
6101         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6102         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6103             my ($model, $macaddr) = split(/\=/, $devconfig);
6104             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6105             my $net = {
6106                 model => $model,
6107                 bridge => "vmbr$ind",
6108                 macaddr => $macaddr,
6109             };
6110             my $netstr = print_net($net);
6111
6112             print $outfd "net$cookie->{netcount}: $netstr\n";
6113             $cookie->{netcount}++;
6114         }
6115     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6116         my ($id, $netstr) = ($1, $2);
6117         my $net = parse_net($netstr);
6118         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6119         $netstr = print_net($net);
6120         print $outfd "$id: $netstr\n";
6121     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
6122         my $virtdev = $1;
6123         my $value = $3;
6124         my $di = parse_drive($virtdev, $value);
6125         if (defined($di->{backup}) && !$di->{backup}) {
6126             print $outfd "#$line";
6127         } elsif ($map->{$virtdev}) {
6128             delete $di->{format}; # format can change on restore
6129             $di->{file} = $map->{$virtdev};
6130             $value = print_drive($di);
6131             print $outfd "$virtdev: $value\n";
6132         } else {
6133             print $outfd $line;
6134         }
6135     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6136         my $vmgenid = $1;
6137         if ($vmgenid ne '0') {
6138             # always generate a new vmgenid if there was a valid one setup
6139             $vmgenid = generate_uuid();
6140         }
6141         print $outfd "vmgenid: $vmgenid\n";
6142     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6143         my ($uuid, $uuid_str);
6144         UUID::generate($uuid);
6145         UUID::unparse($uuid, $uuid_str);
6146         my $smbios1 = parse_smbios1($2);
6147         $smbios1->{uuid} = $uuid_str;
6148         print $outfd $1.print_smbios1($smbios1)."\n";
6149     } else {
6150         print $outfd $line;
6151     }
6152 }
6153
6154 sub scan_volids {
6155     my ($cfg, $vmid) = @_;
6156
6157     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
6158
6159     my $volid_hash = {};
6160     foreach my $storeid (keys %$info) {
6161         foreach my $item (@{$info->{$storeid}}) {
6162             next if !($item->{volid} && $item->{size});
6163             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6164             $volid_hash->{$item->{volid}} = $item;
6165         }
6166     }
6167
6168     return $volid_hash;
6169 }
6170
6171 sub is_volume_in_use {
6172     my ($storecfg, $conf, $skip_drive, $volid) = @_;
6173
6174     my $path = PVE::Storage::path($storecfg, $volid);
6175
6176     my $scan_config = sub {
6177         my ($cref, $snapname) = @_;
6178
6179         foreach my $key (keys %$cref) {
6180             my $value = $cref->{$key};
6181             if (is_valid_drivename($key)) {
6182                 next if $skip_drive && $key eq $skip_drive;
6183                 my $drive = parse_drive($key, $value);
6184                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
6185                 return 1 if $volid eq $drive->{file};
6186                 if ($drive->{file} =~ m!^/!) {
6187                     return 1 if $drive->{file} eq $path;
6188                 } else {
6189                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
6190                     next if !$storeid;
6191                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
6192                     next if !$scfg;
6193                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
6194                 }
6195             }
6196         }
6197
6198         return 0;
6199     };
6200
6201     return 1 if &$scan_config($conf);
6202
6203     undef $skip_drive;
6204
6205     foreach my $snapname (keys %{$conf->{snapshots}}) {
6206         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
6207     }
6208
6209     return 0;
6210 }
6211
6212 sub update_disksize {
6213     my ($drive, $volid_hash) = @_;
6214
6215     my $volid = $drive->{file};
6216     return undef if !defined($volid);
6217
6218     my $oldsize = $drive->{size};
6219     my $newsize = $volid_hash->{$volid}->{size};
6220
6221     if (defined($newsize) && defined($oldsize) && $newsize != $oldsize) {
6222         $drive->{size} = $newsize;
6223
6224         my $old_fmt = PVE::JSONSchema::format_size($oldsize);
6225         my $new_fmt = PVE::JSONSchema::format_size($newsize);
6226
6227         return wantarray ? ($drive, $old_fmt, $new_fmt) : $drive;
6228     }
6229
6230     return undef;
6231 }
6232
6233 sub update_disk_config {
6234     my ($vmid, $conf, $volid_hash) = @_;
6235
6236     my $changes;
6237     my $prefix = "VM $vmid:";
6238
6239     # used and unused disks
6240     my $referenced = {};
6241
6242     # Note: it is allowed to define multiple storages with same path (alias), so
6243     # we need to check both 'volid' and real 'path' (two different volid can point
6244     # to the same path).
6245
6246     my $referencedpath = {};
6247
6248     # update size info
6249     foreach my $opt (keys %$conf) {
6250         if (is_valid_drivename($opt)) {
6251             my $drive = parse_drive($opt, $conf->{$opt});
6252             my $volid = $drive->{file};
6253             next if !$volid;
6254
6255             # mark volid as "in-use" for next step
6256             $referenced->{$volid} = 1;
6257             if ($volid_hash->{$volid} &&
6258                 (my $path = $volid_hash->{$volid}->{path})) {
6259                 $referencedpath->{$path} = 1;
6260             }
6261
6262             next if drive_is_cdrom($drive);
6263             next if !$volid_hash->{$volid};
6264
6265             my ($updated, $old_size, $new_size) = update_disksize($drive, $volid_hash);
6266             if (defined($updated)) {
6267                 $changes = 1;
6268                 $conf->{$opt} = print_drive($updated);
6269                 print "$prefix size of disk '$volid' ($opt) updated from $old_size to $new_size\n";
6270             }
6271         }
6272     }
6273
6274     # remove 'unusedX' entry if volume is used
6275     foreach my $opt (keys %$conf) {
6276         next if $opt !~ m/^unused\d+$/;
6277         my $volid = $conf->{$opt};
6278         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6279         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6280             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6281             $changes = 1;
6282             delete $conf->{$opt};
6283         }
6284
6285         $referenced->{$volid} = 1;
6286         $referencedpath->{$path} = 1 if $path;
6287     }
6288
6289     foreach my $volid (sort keys %$volid_hash) {
6290         next if $volid =~ m/vm-$vmid-state-/;
6291         next if $referenced->{$volid};
6292         my $path = $volid_hash->{$volid}->{path};
6293         next if !$path; # just to be sure
6294         next if $referencedpath->{$path};
6295         $changes = 1;
6296         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6297         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6298         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6299     }
6300
6301     return $changes;
6302 }
6303
6304 sub rescan {
6305     my ($vmid, $nolock, $dryrun) = @_;
6306
6307     my $cfg = PVE::Storage::config();
6308
6309     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6310     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6311     foreach my $stor (keys %{$cfg->{ids}}) {
6312         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6313     }
6314
6315     print "rescan volumes...\n";
6316     my $volid_hash = scan_volids($cfg, $vmid);
6317
6318     my $updatefn =  sub {
6319         my ($vmid) = @_;
6320
6321         my $conf = PVE::QemuConfig->load_config($vmid);
6322
6323         PVE::QemuConfig->check_lock($conf);
6324
6325         my $vm_volids = {};
6326         foreach my $volid (keys %$volid_hash) {
6327             my $info = $volid_hash->{$volid};
6328             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6329         }
6330
6331         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6332
6333         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6334     };
6335
6336     if (defined($vmid)) {
6337         if ($nolock) {
6338             &$updatefn($vmid);
6339         } else {
6340             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6341         }
6342     } else {
6343         my $vmlist = config_list();
6344         foreach my $vmid (keys %$vmlist) {
6345             if ($nolock) {
6346                 &$updatefn($vmid);
6347             } else {
6348                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6349             }
6350         }
6351     }
6352 }
6353
6354 sub restore_vma_archive {
6355     my ($archive, $vmid, $user, $opts, $comp) = @_;
6356
6357     my $readfrom = $archive;
6358
6359     my $cfg = PVE::Storage::config();
6360     my $commands = [];
6361     my $bwlimit = $opts->{bwlimit};
6362
6363     my $dbg_cmdstring = '';
6364     my $add_pipe = sub {
6365         my ($cmd) = @_;
6366         push @$commands, $cmd;
6367         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6368         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6369         $readfrom = '-';
6370     };
6371
6372     my $input = undef;
6373     if ($archive eq '-') {
6374         $input = '<&STDIN';
6375     } else {
6376         # If we use a backup from a PVE defined storage we also consider that
6377         # storage's rate limit:
6378         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6379         if (defined($volid)) {
6380             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6381             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6382             if ($readlimit) {
6383                 print STDERR "applying read rate limit: $readlimit\n";
6384                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6385                 $add_pipe->($cstream);
6386             }
6387         }
6388     }
6389
6390     if ($comp) {
6391         my $cmd;
6392         if ($comp eq 'gzip') {
6393             $cmd = ['zcat', $readfrom];
6394         } elsif ($comp eq 'lzop') {
6395             $cmd = ['lzop', '-d', '-c', $readfrom];
6396         } else {
6397             die "unknown compression method '$comp'\n";
6398         }
6399         $add_pipe->($cmd);
6400     }
6401
6402     my $tmpdir = "/var/tmp/vzdumptmp$$";
6403     rmtree $tmpdir;
6404
6405     # disable interrupts (always do cleanups)
6406     local $SIG{INT} =
6407         local $SIG{TERM} =
6408         local $SIG{QUIT} =
6409         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6410
6411     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6412     POSIX::mkfifo($mapfifo, 0600);
6413     my $fifofh;
6414
6415     my $openfifo = sub {
6416         open($fifofh, '>', $mapfifo) || die $!;
6417     };
6418
6419     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6420
6421     my $oldtimeout;
6422     my $timeout = 5;
6423
6424     my $devinfo = {};
6425
6426     my $rpcenv = PVE::RPCEnvironment::get();
6427
6428     my $conffile = PVE::QemuConfig->config_file($vmid);
6429     my $tmpfn = "$conffile.$$.tmp";
6430
6431     # Note: $oldconf is undef if VM does not exist
6432     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6433     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6434
6435     my %storage_limits;
6436
6437     my $print_devmap = sub {
6438         my $virtdev_hash = {};
6439
6440         my $cfgfn = "$tmpdir/qemu-server.conf";
6441
6442         # we can read the config - that is already extracted
6443         my $fh = IO::File->new($cfgfn, "r") ||
6444             "unable to read qemu-server.conf - $!\n";
6445
6446         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6447         if (-f $fwcfgfn) {
6448             my $pve_firewall_dir = '/etc/pve/firewall';
6449             mkdir $pve_firewall_dir; # make sure the dir exists
6450             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6451         }
6452
6453         while (defined(my $line = <$fh>)) {
6454             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6455                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6456                 die "archive does not contain data for drive '$virtdev'\n"
6457                     if !$devinfo->{$devname};
6458                 if (defined($opts->{storage})) {
6459                     $storeid = $opts->{storage} || 'local';
6460                 } elsif (!$storeid) {
6461                     $storeid = 'local';
6462                 }
6463                 $format = 'raw' if !$format;
6464                 $devinfo->{$devname}->{devname} = $devname;
6465                 $devinfo->{$devname}->{virtdev} = $virtdev;
6466                 $devinfo->{$devname}->{format} = $format;
6467                 $devinfo->{$devname}->{storeid} = $storeid;
6468
6469                 # check permission on storage
6470                 my $pool = $opts->{pool}; # todo: do we need that?
6471                 if ($user ne 'root@pam') {
6472                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6473                 }
6474
6475                 $storage_limits{$storeid} = $bwlimit;
6476
6477                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6478             } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6479                 my $virtdev = $1;
6480                 my $drive = parse_drive($virtdev, $2);
6481                 if (drive_is_cloudinit($drive)) {
6482                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6483                     my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6484                     my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6485
6486                     my $d = {
6487                         format => $format,
6488                         storeid => $opts->{storage} // $storeid,
6489                         size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6490                         file => $drive->{file}, # to make drive_is_cloudinit check possible
6491                         name => "vm-$vmid-cloudinit",
6492                         is_cloudinit => 1,
6493                     };
6494                     $virtdev_hash->{$virtdev} = $d;
6495                 }
6496             }
6497         }
6498
6499         foreach my $key (keys %storage_limits) {
6500             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6501             next if !$limit;
6502             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6503             $storage_limits{$key} = $limit * 1024;
6504         }
6505
6506         foreach my $devname (keys %$devinfo) {
6507             die "found no device mapping information for device '$devname'\n"
6508                 if !$devinfo->{$devname}->{virtdev};
6509         }
6510
6511         # create empty/temp config
6512         if ($oldconf) {
6513             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6514             foreach_drive($oldconf, sub {
6515                 my ($ds, $drive) = @_;
6516
6517                 return if drive_is_cdrom($drive, 1);
6518
6519                 my $volid = $drive->{file};
6520                 return if !$volid || $volid =~ m|^/|;
6521
6522                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6523                 return if !$path || !$owner || ($owner != $vmid);
6524
6525                 # Note: only delete disk we want to restore
6526                 # other volumes will become unused
6527                 if ($virtdev_hash->{$ds}) {
6528                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6529                     if (my $err = $@) {
6530                         warn $err;
6531                     }
6532                 }
6533             });
6534
6535             # delete vmstate files, after the restore we have no snapshots anymore
6536             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6537                 my $snap = $oldconf->{snapshots}->{$snapname};
6538                 if ($snap->{vmstate}) {
6539                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6540                     if (my $err = $@) {
6541                         warn $err;
6542                     }
6543                 }
6544             }
6545         }
6546
6547         my $map = {};
6548         foreach my $virtdev (sort keys %$virtdev_hash) {
6549             my $d = $virtdev_hash->{$virtdev};
6550             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6551             my $storeid = $d->{storeid};
6552             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6553
6554             my $map_opts = '';
6555             if (my $limit = $storage_limits{$storeid}) {
6556                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6557             }
6558
6559             # test if requested format is supported
6560             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6561             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6562             $d->{format} = $defFormat if !$supported;
6563
6564             my $name;
6565             if ($d->{is_cloudinit}) {
6566                 $name = $d->{name};
6567                 $name .= ".$d->{format}" if $d->{format} ne 'raw';
6568             }
6569
6570             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6571             print STDERR "new volume ID is '$volid'\n";
6572             $d->{volid} = $volid;
6573
6574             PVE::Storage::activate_volumes($cfg, [$volid]);
6575
6576             my $write_zeros = 1;
6577             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6578                 $write_zeros = 0;
6579             }
6580
6581             if (!$d->{is_cloudinit}) {
6582                 my $path = PVE::Storage::path($cfg, $volid);
6583
6584                 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6585
6586                 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6587             }
6588             $map->{$virtdev} = $volid;
6589         }
6590
6591         $fh->seek(0, 0) || die "seek failed - $!\n";
6592
6593         my $outfd = new IO::File ($tmpfn, "w") ||
6594             die "unable to write config for VM $vmid\n";
6595
6596         my $cookie = { netcount => 0 };
6597         while (defined(my $line = <$fh>)) {
6598             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6599         }
6600
6601         $fh->close();
6602         $outfd->close();
6603     };
6604
6605     eval {
6606         # enable interrupts
6607         local $SIG{INT} =
6608             local $SIG{TERM} =
6609             local $SIG{QUIT} =
6610             local $SIG{HUP} =
6611             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6612         local $SIG{ALRM} = sub { die "got timeout\n"; };
6613
6614         $oldtimeout = alarm($timeout);
6615
6616         my $parser = sub {
6617             my $line = shift;
6618
6619             print "$line\n";
6620
6621             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6622                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6623                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6624             } elsif ($line =~ m/^CTIME: /) {
6625                 # we correctly received the vma config, so we can disable
6626                 # the timeout now for disk allocation (set to 10 minutes, so
6627                 # that we always timeout if something goes wrong)
6628                 alarm(600);
6629                 &$print_devmap();
6630                 print $fifofh "done\n";
6631                 my $tmp = $oldtimeout || 0;
6632                 $oldtimeout = undef;
6633                 alarm($tmp);
6634                 close($fifofh);
6635             }
6636         };
6637
6638         print "restore vma archive: $dbg_cmdstring\n";
6639         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6640     };
6641     my $err = $@;
6642
6643     alarm($oldtimeout) if $oldtimeout;
6644
6645     my $vollist = [];
6646     foreach my $devname (keys %$devinfo) {
6647         my $volid = $devinfo->{$devname}->{volid};
6648         push @$vollist, $volid if $volid;
6649     }
6650
6651     PVE::Storage::deactivate_volumes($cfg, $vollist);
6652
6653     unlink $mapfifo;
6654
6655     if ($err) {
6656         rmtree $tmpdir;
6657         unlink $tmpfn;
6658
6659         foreach my $devname (keys %$devinfo) {
6660             my $volid = $devinfo->{$devname}->{volid};
6661             next if !$volid;
6662             eval {
6663                 if ($volid =~ m|^/|) {
6664                     unlink $volid || die 'unlink failed\n';
6665                 } else {
6666                     PVE::Storage::vdisk_free($cfg, $volid);
6667                 }
6668                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6669             };
6670             print STDERR "unable to cleanup '$volid' - $@" if $@;
6671         }
6672         die $err;
6673     }
6674
6675     rmtree $tmpdir;
6676
6677     rename($tmpfn, $conffile) ||
6678         die "unable to commit configuration file '$conffile'\n";
6679
6680     PVE::Cluster::cfs_update(); # make sure we read new file
6681
6682     eval { rescan($vmid, 1); };
6683     warn $@ if $@;
6684 }
6685
6686 sub restore_tar_archive {
6687     my ($archive, $vmid, $user, $opts) = @_;
6688
6689     if ($archive ne '-') {
6690         my $firstfile = tar_archive_read_firstfile($archive);
6691         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6692             if $firstfile ne 'qemu-server.conf';
6693     }
6694
6695     my $storecfg = PVE::Storage::config();
6696
6697     # avoid zombie disks when restoring over an existing VM -> cleanup first
6698     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6699     # skiplock=1 because qmrestore has set the 'create' lock itself already
6700     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6701     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6702
6703     my $tocmd = "/usr/lib/qemu-server/qmextract";
6704
6705     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6706     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6707     $tocmd .= ' --prealloc' if $opts->{prealloc};
6708     $tocmd .= ' --info' if $opts->{info};
6709
6710     # tar option "xf" does not autodetect compression when read from STDIN,
6711     # so we pipe to zcat
6712     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6713         PVE::Tools::shellquote("--to-command=$tocmd");
6714
6715     my $tmpdir = "/var/tmp/vzdumptmp$$";
6716     mkpath $tmpdir;
6717
6718     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6719     local $ENV{VZDUMP_VMID} = $vmid;
6720     local $ENV{VZDUMP_USER} = $user;
6721
6722     my $conffile = PVE::QemuConfig->config_file($vmid);
6723     my $tmpfn = "$conffile.$$.tmp";
6724
6725     # disable interrupts (always do cleanups)
6726     local $SIG{INT} =
6727         local $SIG{TERM} =
6728         local $SIG{QUIT} =
6729         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6730
6731     eval {
6732         # enable interrupts
6733         local $SIG{INT} =
6734             local $SIG{TERM} =
6735             local $SIG{QUIT} =
6736             local $SIG{HUP} =
6737             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6738
6739         if ($archive eq '-') {
6740             print "extracting archive from STDIN\n";
6741             run_command($cmd, input => "<&STDIN");
6742         } else {
6743             print "extracting archive '$archive'\n";
6744             run_command($cmd);
6745         }
6746
6747         return if $opts->{info};
6748
6749         # read new mapping
6750         my $map = {};
6751         my $statfile = "$tmpdir/qmrestore.stat";
6752         if (my $fd = IO::File->new($statfile, "r")) {
6753             while (defined (my $line = <$fd>)) {
6754                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6755                     $map->{$1} = $2 if $1;
6756                 } else {
6757                     print STDERR "unable to parse line in statfile - $line\n";
6758                 }
6759             }
6760             $fd->close();
6761         }
6762
6763         my $confsrc = "$tmpdir/qemu-server.conf";
6764
6765         my $srcfd = new IO::File($confsrc, "r") ||
6766             die "unable to open file '$confsrc'\n";
6767
6768         my $outfd = new IO::File ($tmpfn, "w") ||
6769             die "unable to write config for VM $vmid\n";
6770
6771         my $cookie = { netcount => 0 };
6772         while (defined (my $line = <$srcfd>)) {
6773             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6774         }
6775
6776         $srcfd->close();
6777         $outfd->close();
6778     };
6779     if (my $err = $@) {
6780         unlink $tmpfn;
6781         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6782         die $err;
6783     }
6784
6785     rmtree $tmpdir;
6786
6787     rename $tmpfn, $conffile ||
6788         die "unable to commit configuration file '$conffile'\n";
6789
6790     PVE::Cluster::cfs_update(); # make sure we read new file
6791
6792     eval { rescan($vmid, 1); };
6793     warn $@ if $@;
6794 };
6795
6796 sub foreach_storage_used_by_vm {
6797     my ($conf, $func) = @_;
6798
6799     my $sidhash = {};
6800
6801     foreach_drive($conf, sub {
6802         my ($ds, $drive) = @_;
6803         return if drive_is_cdrom($drive);
6804
6805         my $volid = $drive->{file};
6806
6807         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6808         $sidhash->{$sid} = $sid if $sid;
6809     });
6810
6811     foreach my $sid (sort keys %$sidhash) {
6812         &$func($sid);
6813     }
6814 }
6815
6816 my $qemu_snap_storage = {
6817     rbd => 1,
6818 };
6819 sub do_snapshots_with_qemu {
6820     my ($storecfg, $volid) = @_;
6821
6822     my $storage_name = PVE::Storage::parse_volume_id($volid);
6823     my $scfg = $storecfg->{ids}->{$storage_name};
6824
6825     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6826         return 1;
6827     }
6828
6829     if ($volid =~ m/\.(qcow2|qed)$/){
6830         return 1;
6831     }
6832
6833     return undef;
6834 }
6835
6836 sub qga_check_running {
6837     my ($vmid, $nowarn) = @_;
6838
6839     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6840     if ($@) {
6841         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6842         return 0;
6843     }
6844     return 1;
6845 }
6846
6847 sub template_create {
6848     my ($vmid, $conf, $disk) = @_;
6849
6850     my $storecfg = PVE::Storage::config();
6851
6852     foreach_drive($conf, sub {
6853         my ($ds, $drive) = @_;
6854
6855         return if drive_is_cdrom($drive);
6856         return if $disk && $ds ne $disk;
6857
6858         my $volid = $drive->{file};
6859         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6860
6861         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6862         $drive->{file} = $voliddst;
6863         $conf->{$ds} = print_drive($drive);
6864         PVE::QemuConfig->write_config($vmid, $conf);
6865     });
6866 }
6867
6868 sub convert_iscsi_path {
6869     my ($path) = @_;
6870
6871     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6872         my $portal = $1;
6873         my $target = $2;
6874         my $lun = $3;
6875
6876         my $initiator_name = get_initiator_name();
6877
6878         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6879                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6880     }
6881
6882     die "cannot convert iscsi path '$path', unkown format\n";
6883 }
6884
6885 sub qemu_img_convert {
6886     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6887
6888     my $storecfg = PVE::Storage::config();
6889     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6890     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6891
6892     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6893
6894     my $cachemode;
6895     my $src_path;
6896     my $src_is_iscsi = 0;
6897     my $src_format;
6898
6899     if ($src_storeid) {
6900         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6901         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6902         $src_format = qemu_img_format($src_scfg, $src_volname);
6903         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6904         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6905         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6906     } elsif (-f $src_volid) {
6907         $src_path = $src_volid;
6908         if ($src_path =~ m/\.($QEMU_FORMAT_RE)$/) {
6909             $src_format = $1;
6910         }
6911     }
6912
6913     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6914
6915     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6916     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6917     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6918     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6919
6920     my $cmd = [];
6921     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6922     push @$cmd, '-l', "snapshot.name=$snapname"
6923         if $snapname && $src_format && $src_format eq "qcow2";
6924     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6925     push @$cmd, '-T', $cachemode if defined($cachemode);
6926
6927     if ($src_is_iscsi) {
6928         push @$cmd, '--image-opts';
6929         $src_path = convert_iscsi_path($src_path);
6930     } elsif ($src_format) {
6931         push @$cmd, '-f', $src_format;
6932     }
6933
6934     if ($dst_is_iscsi) {
6935         push @$cmd, '--target-image-opts';
6936         $dst_path = convert_iscsi_path($dst_path);
6937     } else {
6938         push @$cmd, '-O', $dst_format;
6939     }
6940
6941     push @$cmd, $src_path;
6942
6943     if (!$dst_is_iscsi && $is_zero_initialized) {
6944         push @$cmd, "zeroinit:$dst_path";
6945     } else {
6946         push @$cmd, $dst_path;
6947     }
6948
6949     my $parser = sub {
6950         my $line = shift;
6951         if($line =~ m/\((\S+)\/100\%\)/){
6952             my $percent = $1;
6953             my $transferred = int($size * $percent / 100);
6954             my $remaining = $size - $transferred;
6955
6956             print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6957         }
6958
6959     };
6960
6961     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6962     my $err = $@;
6963     die "copy failed: $err" if $err;
6964 }
6965
6966 sub qemu_img_format {
6967     my ($scfg, $volname) = @_;
6968
6969     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6970         return $1;
6971     } else {
6972         return "raw";
6973     }
6974 }
6975
6976 sub qemu_drive_mirror {
6977     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6978
6979     $jobs = {} if !$jobs;
6980
6981     my $qemu_target;
6982     my $format;
6983     $jobs->{"drive-$drive"} = {};
6984
6985     if ($dst_volid =~ /^nbd:/) {
6986         $qemu_target = $dst_volid;
6987         $format = "nbd";
6988     } else {
6989         my $storecfg = PVE::Storage::config();
6990         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6991
6992         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6993
6994         $format = qemu_img_format($dst_scfg, $dst_volname);
6995
6996         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6997
6998         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6999     }
7000
7001     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7002     $opts->{format} = $format if $format;
7003
7004     if (defined($bwlimit)) {
7005         $opts->{speed} = $bwlimit * 1024;
7006         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7007     } else {
7008         print "drive mirror is starting for drive-$drive\n";
7009     }
7010
7011     # if a job already runs for this device we get an error, catch it for cleanup
7012     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7013     if (my $err = $@) {
7014         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7015         warn "$@\n" if $@;
7016         die "mirroring error: $err\n";
7017     }
7018
7019     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
7020 }
7021
7022 sub qemu_drive_mirror_monitor {
7023     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
7024
7025     eval {
7026         my $err_complete = 0;
7027
7028         while (1) {
7029             die "storage migration timed out\n" if $err_complete > 300;
7030
7031             my $stats = mon_cmd($vmid, "query-block-jobs");
7032
7033             my $running_mirror_jobs = {};
7034             foreach my $stat (@$stats) {
7035                 next if $stat->{type} ne 'mirror';
7036                 $running_mirror_jobs->{$stat->{device}} = $stat;
7037             }
7038
7039             my $readycounter = 0;
7040
7041             foreach my $job (keys %$jobs) {
7042
7043                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
7044                     print "$job : finished\n";
7045                     delete $jobs->{$job};
7046                     next;
7047                 }
7048
7049                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
7050
7051                 my $busy = $running_mirror_jobs->{$job}->{busy};
7052                 my $ready = $running_mirror_jobs->{$job}->{ready};
7053                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
7054                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
7055                     my $remaining = $total - $transferred;
7056                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7057
7058                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
7059                 }
7060
7061                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
7062             }
7063
7064             last if scalar(keys %$jobs) == 0;
7065
7066             if ($readycounter == scalar(keys %$jobs)) {
7067                 print "all mirroring jobs are ready \n";
7068                 last if $skipcomplete; #do the complete later
7069
7070                 if ($vmiddst && $vmiddst != $vmid) {
7071                     my $agent_running = $qga && qga_check_running($vmid);
7072                     if ($agent_running) {
7073                         print "freeze filesystem\n";
7074                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7075                     } else {
7076                         print "suspend vm\n";
7077                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7078                     }
7079
7080                     # if we clone a disk for a new target vm, we don't switch the disk
7081                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7082
7083                     if ($agent_running) {
7084                         print "unfreeze filesystem\n";
7085                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7086                     } else {
7087                         print "resume vm\n";
7088                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
7089                     }
7090
7091                     last;
7092                 } else {
7093
7094                     foreach my $job (keys %$jobs) {
7095                         # try to switch the disk if source and destination are on the same guest
7096                         print "$job: Completing block job...\n";
7097
7098                         eval { mon_cmd($vmid, "block-job-complete", device => $job) };
7099                         if ($@ =~ m/cannot be completed/) {
7100                             print "$job: Block job cannot be completed, try again.\n";
7101                             $err_complete++;
7102                         }else {
7103                             print "$job: Completed successfully.\n";
7104                             $jobs->{$job}->{complete} = 1;
7105                         }
7106                     }
7107                 }
7108             }
7109             sleep 1;
7110         }
7111     };
7112     my $err = $@;
7113
7114     if ($err) {
7115         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7116         die "mirroring error: $err";
7117     }
7118
7119 }
7120
7121 sub qemu_blockjobs_cancel {
7122     my ($vmid, $jobs) = @_;
7123
7124     foreach my $job (keys %$jobs) {
7125         print "$job: Cancelling block job\n";
7126         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7127         $jobs->{$job}->{cancel} = 1;
7128     }
7129
7130     while (1) {
7131         my $stats = mon_cmd($vmid, "query-block-jobs");
7132
7133         my $running_jobs = {};
7134         foreach my $stat (@$stats) {
7135             $running_jobs->{$stat->{device}} = $stat;
7136         }
7137
7138         foreach my $job (keys %$jobs) {
7139
7140             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7141                 print "$job: Done.\n";
7142                 delete $jobs->{$job};
7143             }
7144         }
7145
7146         last if scalar(keys %$jobs) == 0;
7147
7148         sleep 1;
7149     }
7150 }
7151
7152 sub clone_disk {
7153     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7154         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
7155
7156     my $newvolid;
7157
7158     if (!$full) {
7159         print "create linked clone of drive $drivename ($drive->{file})\n";
7160         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7161         push @$newvollist, $newvolid;
7162     } else {
7163
7164         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7165         $storeid = $storage if $storage;
7166
7167         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7168         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
7169
7170         print "create full clone of drive $drivename ($drive->{file})\n";
7171         my $name = undef;
7172         if (drive_is_cloudinit($drive)) {
7173             $name = "vm-$newvmid-cloudinit";
7174             $name .= ".$dst_format" if $dst_format ne 'raw';
7175             $snapname = undef;
7176             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7177         }
7178         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
7179         push @$newvollist, $newvolid;
7180
7181         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7182
7183         if (drive_is_cloudinit($drive)) {
7184             goto no_data_clone;
7185         }
7186
7187         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7188         if (!$running || $snapname) {
7189             # TODO: handle bwlimits
7190             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7191         } else {
7192
7193             my $kvmver = get_running_qemu_version ($vmid);
7194             if (!min_version($kvmver, 2, 7)) {
7195                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7196                     if $drive->{iothread};
7197             }
7198
7199             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga, $bwlimit);
7200         }
7201     }
7202
7203 no_data_clone:
7204     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
7205
7206     my $disk = $drive;
7207     $disk->{format} = undef;
7208     $disk->{file} = $newvolid;
7209     $disk->{size} = $size;
7210
7211     return $disk;
7212 }
7213
7214 sub get_running_qemu_version {
7215     my ($vmid) = @_;
7216     my $res = mon_cmd($vmid, "query-version");
7217     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7218 }
7219
7220 sub qemu_use_old_bios_files {
7221     my ($machine_type) = @_;
7222
7223     return if !$machine_type;
7224
7225     my $use_old_bios_files = undef;
7226
7227     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7228         $machine_type = $1;
7229         $use_old_bios_files = 1;
7230     } else {
7231         my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
7232         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7233         # load new efi bios files on migration. So this hack is required to allow
7234         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7235         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7236         $use_old_bios_files = !min_version($version, 2, 4);
7237     }
7238
7239     return ($use_old_bios_files, $machine_type);
7240 }
7241
7242 sub create_efidisk($$$$$) {
7243     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7244
7245     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7246     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7247
7248     my $vars_size_b = -s $ovmf_vars;
7249     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7250     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7251     PVE::Storage::activate_volumes($storecfg, [$volid]);
7252
7253     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7254     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7255
7256     return ($volid, $size/1024);
7257 }
7258
7259 sub vm_iothreads_list {
7260     my ($vmid) = @_;
7261
7262     my $res = mon_cmd($vmid, 'query-iothreads');
7263
7264     my $iothreads = {};
7265     foreach my $iothread (@$res) {
7266         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7267     }
7268
7269     return $iothreads;
7270 }
7271
7272 sub scsihw_infos {
7273     my ($conf, $drive) = @_;
7274
7275     my $maxdev = 0;
7276
7277     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7278         $maxdev = 7;
7279     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7280         $maxdev = 1;
7281     } else {
7282         $maxdev = 256;
7283     }
7284
7285     my $controller = int($drive->{index} / $maxdev);
7286     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
7287
7288     return ($maxdev, $controller, $controller_prefix);
7289 }
7290
7291 sub add_hyperv_enlightenments {
7292     my ($cpuFlags, $winversion, $machine_version, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
7293
7294     return if $winversion < 6;
7295     return if $bios && $bios eq 'ovmf' && $winversion < 8;
7296
7297     if ($gpu_passthrough || defined($hv_vendor_id)) {
7298         $hv_vendor_id //= 'proxmox';
7299         push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
7300     }
7301
7302     if (min_version($machine_version, 2, 3)) {
7303         push @$cpuFlags , 'hv_spinlocks=0x1fff';
7304         push @$cpuFlags , 'hv_vapic';
7305         push @$cpuFlags , 'hv_time';
7306     } else {
7307         push @$cpuFlags , 'hv_spinlocks=0xffff';
7308     }
7309
7310     if (min_version($machine_version, 2, 6)) {
7311         push @$cpuFlags , 'hv_reset';
7312         push @$cpuFlags , 'hv_vpindex';
7313         push @$cpuFlags , 'hv_runtime';
7314     }
7315
7316     if ($winversion >= 7) {
7317         push @$cpuFlags , 'hv_relaxed';
7318
7319         if (min_version($machine_version, 2, 12)) {
7320             push @$cpuFlags , 'hv_synic';
7321             push @$cpuFlags , 'hv_stimer';
7322         }
7323
7324         if (min_version($machine_version, 3, 1)) {
7325             push @$cpuFlags , 'hv_ipi';
7326         }
7327     }
7328 }
7329
7330 sub windows_version {
7331     my ($ostype) = @_;
7332
7333     return 0 if !$ostype;
7334
7335     my $winversion = 0;
7336
7337     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7338         $winversion = 5;
7339     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7340         $winversion = 6;
7341     } elsif ($ostype =~ m/^win(\d+)$/) {
7342         $winversion = $1;
7343     }
7344
7345     return $winversion;
7346 }
7347
7348 sub resolve_dst_disk_format {
7349         my ($storecfg, $storeid, $src_volname, $format) = @_;
7350         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7351
7352         if (!$format) {
7353             # if no target format is specified, use the source disk format as hint
7354             if ($src_volname) {
7355                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7356                 $format = qemu_img_format($scfg, $src_volname);
7357             } else {
7358                 return $defFormat;
7359             }
7360         }
7361
7362         # test if requested format is supported - else use default
7363         my $supported = grep { $_ eq $format } @$validFormats;
7364         $format = $defFormat if !$supported;
7365         return $format;
7366 }
7367
7368 sub resolve_first_disk {
7369     my $conf = shift;
7370     my @disks = PVE::QemuServer::valid_drive_names();
7371     my $firstdisk;
7372     foreach my $ds (reverse @disks) {
7373         next if !$conf->{$ds};
7374         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
7375         next if PVE::QemuServer::drive_is_cdrom($disk);
7376         $firstdisk = $ds;
7377     }
7378     return $firstdisk;
7379 }
7380
7381 # NOTE: if this logic changes, please update docs & possibly gui logic
7382 sub find_vmstate_storage {
7383     my ($conf, $storecfg) = @_;
7384
7385     # first, return storage from conf if set
7386     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7387
7388     my ($target, $shared, $local);
7389
7390     foreach_storage_used_by_vm($conf, sub {
7391         my ($sid) = @_;
7392         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7393         my $dst = $scfg->{shared} ? \$shared : \$local;
7394         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7395     });
7396
7397     # second, use shared storage where VM has at least one disk
7398     # third, use local storage where VM has at least one disk
7399     # fall back to local storage
7400     $target = $shared // $local // 'local';
7401
7402     return $target;
7403 }
7404
7405 sub generate_uuid {
7406     my ($uuid, $uuid_str);
7407     UUID::generate($uuid);
7408     UUID::unparse($uuid, $uuid_str);
7409     return $uuid_str;
7410 }
7411
7412 sub generate_smbios1_uuid {
7413     return "uuid=".generate_uuid();
7414 }
7415
7416 sub nbd_stop {
7417     my ($vmid) = @_;
7418
7419     mon_cmd($vmid, 'nbd-server-stop');
7420 }
7421
7422 sub create_reboot_request {
7423     my ($vmid) = @_;
7424     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7425         or die "failed to create reboot trigger file: $!\n";
7426     close($fh);
7427 }
7428
7429 sub clear_reboot_request {
7430     my ($vmid) = @_;
7431     my $path = "/run/qemu-server/$vmid.reboot";
7432     my $res = 0;
7433
7434     $res = unlink($path);
7435     die "could not remove reboot request for $vmid: $!"
7436         if !$res && $! != POSIX::ENOENT;
7437
7438     return $res;
7439 }
7440
7441 # bash completion helper
7442
7443 sub complete_backup_archives {
7444     my ($cmdname, $pname, $cvalue) = @_;
7445
7446     my $cfg = PVE::Storage::config();
7447
7448     my $storeid;
7449
7450     if ($cvalue =~ m/^([^:]+):/) {
7451         $storeid = $1;
7452     }
7453
7454     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7455
7456     my $res = [];
7457     foreach my $id (keys %$data) {
7458         foreach my $item (@{$data->{$id}}) {
7459             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
7460             push @$res, $item->{volid} if defined($item->{volid});
7461         }
7462     }
7463
7464     return $res;
7465 }
7466
7467 my $complete_vmid_full = sub {
7468     my ($running) = @_;
7469
7470     my $idlist = vmstatus();
7471
7472     my $res = [];
7473
7474     foreach my $id (keys %$idlist) {
7475         my $d = $idlist->{$id};
7476         if (defined($running)) {
7477             next if $d->{template};
7478             next if $running && $d->{status} ne 'running';
7479             next if !$running && $d->{status} eq 'running';
7480         }
7481         push @$res, $id;
7482
7483     }
7484     return $res;
7485 };
7486
7487 sub complete_vmid {
7488     return &$complete_vmid_full();
7489 }
7490
7491 sub complete_vmid_stopped {
7492     return &$complete_vmid_full(0);
7493 }
7494
7495 sub complete_vmid_running {
7496     return &$complete_vmid_full(1);
7497 }
7498
7499 sub complete_storage {
7500
7501     my $cfg = PVE::Storage::config();
7502     my $ids = $cfg->{ids};
7503
7504     my $res = [];
7505     foreach my $sid (keys %$ids) {
7506         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7507         next if !$ids->{$sid}->{content}->{images};
7508         push @$res, $sid;
7509     }
7510
7511     return $res;
7512 }
7513
7514 sub complete_migration_storage {
7515     my ($cmd, $param, $current_value, $all_args) = @_;
7516
7517     my $targetnode = @$all_args[1];
7518
7519     my $cfg = PVE::Storage::config();
7520     my $ids = $cfg->{ids};
7521
7522     my $res = [];
7523     foreach my $sid (keys %$ids) {
7524         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7525         next if !$ids->{$sid}->{content}->{images};
7526         push @$res, $sid;
7527     }
7528
7529     return $res;
7530 }
7531
7532 1;