PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::DataCenterConfig;
  31 use PVE::Exception qw(raise raise_param_exc);
  32 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  33 use PVE::INotify;
  34 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  35 use PVE::ProcFSTools;
  36 use PVE::RPCEnvironment;
  37 use PVE::Storage;
  38 use PVE::SysFSTools;
  39 use PVE::Systemd;
  40 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  41
  42 use PVE::QMPClient;
  43 use PVE::QemuConfig;
  44 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  45 use PVE::QemuServer::Cloudinit;
  46 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  47 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom parse_drive print_drive);
  48 use PVE::QemuServer::Machine;
  49 use PVE::QemuServer::Memory;
  50 use PVE::QemuServer::Monitor qw(mon_cmd);
  51 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  52 use PVE::QemuServer::USB qw(parse_usb_device);
  53
  54 my $have_sdn;
  55 eval {
  56     require PVE::Network::SDN::Zones;
  57     $have_sdn = 1;
  58 };
  59
  60 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  61 my $OVMF = {
  62     x86_64 => [
  63         "$EDK2_FW_BASE/OVMF_CODE.fd",
  64         "$EDK2_FW_BASE/OVMF_VARS.fd"
  65     ],
  66     aarch64 => [
  67         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  68         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  69     ],
  70 };
  71
  72 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  73
  74 # Note about locking: we use flock on the config file protect
  75 # against concurent actions.
  76 # Aditionaly, we have a 'lock' setting in the config file. This
  77 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  78 # allowed when such lock is set. But you can ignore this kind of
  79 # lock with the --skiplock flag.
  80
  81 cfs_register_file('/qemu-server/',
  82                   \&parse_vm_config,
  83                   \&write_vm_config);
  84
  85 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  86     description => "Some command save/restore state from this location.",
  87     type => 'string',
  88     maxLength => 128,
  89     optional => 1,
  90 });
  91
  92 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  93         description => "Specifies the Qemu machine type.",
  94         type => 'string',
  95         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
  96         maxLength => 40,
  97         optional => 1,
  98 });
  99
 100
 101 sub map_storage {
 102     my ($map, $source) = @_;
 103
 104     return $source if !defined($map);
 105
 106     return $map->{entries}->{$source}
 107         if $map->{entries} && defined($map->{entries}->{$source});
 108
 109     return $map->{default} if $map->{default};
 110
 111     # identity (fallback)
 112     return $source;
 113 }
 114
 115 PVE::JSONSchema::register_standard_option('pve-targetstorage', {
 116     description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
 117     type => 'string',
 118     format => 'storagepair-list',
 119     optional => 1,
 120 });
 121
 122 #no warnings 'redefine';
 123
 124 sub cgroups_write {
 125    my ($controller, $vmid, $option, $value) = @_;
 126
 127    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 128    PVE::ProcFSTools::write_proc_entry($path, $value);
 129
 130 }
 131
 132 my $nodename_cache;
 133 sub nodename {
 134     $nodename_cache //= PVE::INotify::nodename();
 135     return $nodename_cache;
 136 }
 137
 138 my $watchdog_fmt = {
 139     model => {
 140         default_key => 1,
 141         type => 'string',
 142         enum => [qw(i6300esb ib700)],
 143         description => "Watchdog type to emulate.",
 144         default => 'i6300esb',
 145         optional => 1,
 146     },
 147     action => {
 148         type => 'string',
 149         enum => [qw(reset shutdown poweroff pause debug none)],
 150         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 151         optional => 1,
 152     },
 153 };
 154 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 155
 156 my $agent_fmt = {
 157     enabled => {
 158         description => "Enable/disable Qemu GuestAgent.",
 159         type => 'boolean',
 160         default => 0,
 161         default_key => 1,
 162     },
 163     fstrim_cloned_disks => {
 164         description => "Run fstrim after cloning/moving a disk.",
 165         type => 'boolean',
 166         optional => 1,
 167         default => 0
 168     },
 169     type => {
 170         description => "Select the agent type",
 171         type => 'string',
 172         default => 'virtio',
 173         optional => 1,
 174         enum => [qw(virtio isa)],
 175     },
 176 };
 177
 178 my $vga_fmt = {
 179     type => {
 180         description => "Select the VGA type.",
 181         type => 'string',
 182         default => 'std',
 183         optional => 1,
 184         default_key => 1,
 185         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 186     },
 187     memory => {
 188         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 189         type => 'integer',
 190         optional => 1,
 191         minimum => 4,
 192         maximum => 512,
 193     },
 194 };
 195
 196 my $ivshmem_fmt = {
 197     size => {
 198         type => 'integer',
 199         minimum => 1,
 200         description => "The size of the file in MB.",
 201     },
 202     name => {
 203         type => 'string',
 204         pattern => '[a-zA-Z0-9\-]+',
 205         optional => 1,
 206         format_description => 'string',
 207         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 208     },
 209 };
 210
 211 my $audio_fmt = {
 212     device => {
 213         type => 'string',
 214         enum => [qw(ich9-intel-hda intel-hda AC97)],
 215         description =>  "Configure an audio device."
 216     },
 217     driver =>  {
 218         type => 'string',
 219         enum => ['spice'],
 220         default => 'spice',
 221         optional => 1,
 222         description => "Driver backend for the audio device."
 223     },
 224 };
 225
 226 my $spice_enhancements_fmt = {
 227     foldersharing => {
 228         type => 'boolean',
 229         optional => 1,
 230         default => '0',
 231         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 232     },
 233     videostreaming =>  {
 234         type => 'string',
 235         enum => ['off', 'all', 'filter'],
 236         default => 'off',
 237         optional => 1,
 238         description => "Enable video streaming. Uses compression for detected video streams."
 239     },
 240 };
 241
 242 my $rng_fmt = {
 243     source => {
 244         type => 'string',
 245         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 246         default_key => 1,
 247         description => "The file on the host to gather entropy from. In most"
 248                      . " cases /dev/urandom should be preferred over /dev/random"
 249                      . " to avoid entropy-starvation issues on the host. Using"
 250                      . " urandom does *not* decrease security in any meaningful"
 251                      . " way, as it's still seeded from real entropy, and the"
 252                      . " bytes provided will most likely be mixed with real"
 253                      . " entropy on the guest as well. /dev/hwrng can be used"
 254                      . " to pass through a hardware RNG from the host.",
 255     },
 256     max_bytes => {
 257         type => 'integer',
 258         description => "Maximum bytes of entropy injected into the guest every"
 259                      . " 'period' milliseconds. Prefer a lower value when using"
 260                      . " /dev/random as source. Use 0 to disable limiting"
 261                      . " (potentially dangerous!).",
 262         optional => 1,
 263
 264         # default is 1 KiB/s, provides enough entropy to the guest to avoid
 265         # boot-starvation issues (e.g. systemd etc...) while allowing no chance
 266         # of overwhelming the host, provided we're reading from /dev/urandom
 267         default => 1024,
 268     },
 269     period => {
 270         type => 'integer',
 271         description => "Every 'period' milliseconds the entropy-injection quota"
 272                      . " is reset, allowing the guest to retrieve another"
 273                      . " 'max_bytes' of entropy.",
 274         optional => 1,
 275         default => 1000,
 276     },
 277 };
 278
 279 my $confdesc = {
 280     onboot => {
 281         optional => 1,
 282         type => 'boolean',
 283         description => "Specifies whether a VM will be started during system bootup.",
 284         default => 0,
 285     },
 286     autostart => {
 287         optional => 1,
 288         type => 'boolean',
 289         description => "Automatic restart after crash (currently ignored).",
 290         default => 0,
 291     },
 292     hotplug => {
 293         optional => 1,
 294         type => 'string', format => 'pve-hotplug-features',
 295         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 296         default => 'network,disk,usb',
 297     },
 298     reboot => {
 299         optional => 1,
 300         type => 'boolean',
 301         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 302         default => 1,
 303     },
 304     lock => {
 305         optional => 1,
 306         type => 'string',
 307         description => "Lock/unlock the VM.",
 308         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 309     },
 310     cpulimit => {
 311         optional => 1,
 312         type => 'number',
 313         description => "Limit of CPU usage.",
 314         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 315         minimum => 0,
 316         maximum => 128,
 317         default => 0,
 318     },
 319     cpuunits => {
 320         optional => 1,
 321         type => 'integer',
 322         description => "CPU weight for a VM.",
 323         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 324         minimum => 2,
 325         maximum => 262144,
 326         default => 1024,
 327     },
 328     memory => {
 329         optional => 1,
 330         type => 'integer',
 331         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 332         minimum => 16,
 333         default => 512,
 334     },
 335     balloon => {
 336         optional => 1,
 337         type => 'integer',
 338         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 339         minimum => 0,
 340     },
 341     shares => {
 342         optional => 1,
 343         type => 'integer',
 344         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 345         minimum => 0,
 346         maximum => 50000,
 347         default => 1000,
 348     },
 349     keyboard => {
 350         optional => 1,
 351         type => 'string',
 352         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 353                        "It should not be necessary to set it.",
 354         enum => PVE::Tools::kvmkeymaplist(),
 355         default => undef,
 356     },
 357     name => {
 358         optional => 1,
 359         type => 'string', format => 'dns-name',
 360         description => "Set a name for the VM. Only used on the configuration web interface.",
 361     },
 362     scsihw => {
 363         optional => 1,
 364         type => 'string',
 365         description => "SCSI controller model",
 366         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 367         default => 'lsi',
 368     },
 369     description => {
 370         optional => 1,
 371         type => 'string',
 372         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 373     },
 374     ostype => {
 375         optional => 1,
 376         type => 'string',
 377         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 378         description => "Specify guest operating system.",
 379         verbose_description => <<EODESC,
 380 Specify guest operating system. This is used to enable special
 381 optimization/features for specific operating systems:
 382
 383 [horizontal]
 384 other;; unspecified OS
 385 wxp;; Microsoft Windows XP
 386 w2k;; Microsoft Windows 2000
 387 w2k3;; Microsoft Windows 2003
 388 w2k8;; Microsoft Windows 2008
 389 wvista;; Microsoft Windows Vista
 390 win7;; Microsoft Windows 7
 391 win8;; Microsoft Windows 8/2012/2012r2
 392 win10;; Microsoft Windows 10/2016
 393 l24;; Linux 2.4 Kernel
 394 l26;; Linux 2.6 - 5.X Kernel
 395 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 396 EODESC
 397     },
 398     boot => {
 399         optional => 1,
 400         type => 'string', format => 'pve-qm-boot',
 401         description => "Specify guest boot order. Use with 'order=', usage with"
 402                      . " no key or 'legacy=' is deprecated.",
 403     },
 404     bootdisk => {
 405         optional => 1,
 406         type => 'string', format => 'pve-qm-bootdisk',
 407         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 408         pattern => '(ide|sata|scsi|virtio)\d+',
 409     },
 410     smp => {
 411         optional => 1,
 412         type => 'integer',
 413         description => "The number of CPUs. Please use option -sockets instead.",
 414         minimum => 1,
 415         default => 1,
 416     },
 417     sockets => {
 418         optional => 1,
 419         type => 'integer',
 420         description => "The number of CPU sockets.",
 421         minimum => 1,
 422         default => 1,
 423     },
 424     cores => {
 425         optional => 1,
 426         type => 'integer',
 427         description => "The number of cores per socket.",
 428         minimum => 1,
 429         default => 1,
 430     },
 431     numa => {
 432         optional => 1,
 433         type => 'boolean',
 434         description => "Enable/disable NUMA.",
 435         default => 0,
 436     },
 437     hugepages => {
 438         optional => 1,
 439         type => 'string',
 440         description => "Enable/disable hugepages memory.",
 441         enum => [qw(any 2 1024)],
 442     },
 443     keephugepages => {
 444         optional => 1,
 445         type => 'boolean',
 446         default => 0,
 447         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 448             ." after VM shutdown and can be used for subsequent starts.",
 449     },
 450     vcpus => {
 451         optional => 1,
 452         type => 'integer',
 453         description => "Number of hotplugged vcpus.",
 454         minimum => 1,
 455         default => 0,
 456     },
 457     acpi => {
 458         optional => 1,
 459         type => 'boolean',
 460         description => "Enable/disable ACPI.",
 461         default => 1,
 462     },
 463     agent => {
 464         optional => 1,
 465         description => "Enable/disable Qemu GuestAgent and its properties.",
 466         type => 'string',
 467         format => $agent_fmt,
 468     },
 469     kvm => {
 470         optional => 1,
 471         type => 'boolean',
 472         description => "Enable/disable KVM hardware virtualization.",
 473         default => 1,
 474     },
 475     tdf => {
 476         optional => 1,
 477         type => 'boolean',
 478         description => "Enable/disable time drift fix.",
 479         default => 0,
 480     },
 481     localtime => {
 482         optional => 1,
 483         type => 'boolean',
 484         description => "Set the real time clock to local time. This is enabled by default if ostype"
 485             ." indicates a Microsoft OS.",
 486     },
 487     freeze => {
 488         optional => 1,
 489         type => 'boolean',
 490         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 491     },
 492     vga => {
 493         optional => 1,
 494         type => 'string', format => $vga_fmt,
 495         description => "Configure the VGA hardware.",
 496         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 497             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 498             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 499             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 500             ." display server. For win* OS you can select how many independent displays you want,"
 501             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 502             ." using a serial device as terminal.",
 503     },
 504     watchdog => {
 505         optional => 1,
 506         type => 'string', format => 'pve-qm-watchdog',
 507         description => "Create a virtual hardware watchdog device.",
 508         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 509             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 510             ." else the watchdog will reset the guest (or execute the respective action specified)",
 511     },
 512     startdate => {
 513         optional => 1,
 514         type => 'string',
 515         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 516         description => "Set the initial date of the real time clock. Valid format for date are:"
 517             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 518         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 519         default => 'now',
 520     },
 521     startup =>  get_standard_option('pve-startup-order'),
 522     template => {
 523         optional => 1,
 524         type => 'boolean',
 525         description => "Enable/disable Template.",
 526         default => 0,
 527     },
 528     args => {
 529         optional => 1,
 530         type => 'string',
 531         description => "Arbitrary arguments passed to kvm.",
 532         verbose_description => <<EODESCR,
 533 Arbitrary arguments passed to kvm, for example:
 534
 535 args: -no-reboot -no-hpet
 536
 537 NOTE: this option is for experts only.
 538 EODESCR
 539     },
 540     tablet => {
 541         optional => 1,
 542         type => 'boolean',
 543         default => 1,
 544         description => "Enable/disable the USB tablet device.",
 545         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 546             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 547             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 548             ." may consider disabling this to save some context switches. This is turned off by"
 549             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 550     },
 551     migrate_speed => {
 552         optional => 1,
 553         type => 'integer',
 554         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 555         minimum => 0,
 556         default => 0,
 557     },
 558     migrate_downtime => {
 559         optional => 1,
 560         type => 'number',
 561         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 562         minimum => 0,
 563         default => 0.1,
 564     },
 565     cdrom => {
 566         optional => 1,
 567         type => 'string', format => 'pve-qm-ide',
 568         typetext => '<volume>',
 569         description => "This is an alias for option -ide2",
 570     },
 571     cpu => {
 572         optional => 1,
 573         description => "Emulated CPU type.",
 574         type => 'string',
 575         format => 'pve-vm-cpu-conf',
 576     },
 577     parent => get_standard_option('pve-snapshot-name', {
 578         optional => 1,
 579         description => "Parent snapshot name. This is used internally, and should not be modified.",
 580     }),
 581     snaptime => {
 582         optional => 1,
 583         description => "Timestamp for snapshots.",
 584         type => 'integer',
 585         minimum => 0,
 586     },
 587     vmstate => {
 588         optional => 1,
 589         type => 'string', format => 'pve-volume-id',
 590         description => "Reference to a volume which stores the VM state. This is used internally"
 591             ." for snapshots.",
 592     },
 593     vmstatestorage => get_standard_option('pve-storage-id', {
 594         description => "Default storage for VM state volumes/files.",
 595         optional => 1,
 596     }),
 597     runningmachine => get_standard_option('pve-qemu-machine', {
 598         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 599             ." for snapshots.",
 600     }),
 601     runningcpu => {
 602         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 603             ." internally for snapshots.",
 604         optional => 1,
 605         type => 'string',
 606         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 607         format_description => 'QEMU -cpu parameter'
 608     },
 609     machine => get_standard_option('pve-qemu-machine'),
 610     arch => {
 611         description => "Virtual processor architecture. Defaults to the host.",
 612         optional => 1,
 613         type => 'string',
 614         enum => [qw(x86_64 aarch64)],
 615     },
 616     smbios1 => {
 617         description => "Specify SMBIOS type 1 fields.",
 618         type => 'string', format => 'pve-qm-smbios1',
 619         maxLength => 512,
 620         optional => 1,
 621     },
 622     protection => {
 623         optional => 1,
 624         type => 'boolean',
 625         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 626             ." remove disk operations.",
 627         default => 0,
 628     },
 629     bios => {
 630         optional => 1,
 631         type => 'string',
 632         enum => [ qw(seabios ovmf) ],
 633         description => "Select BIOS implementation.",
 634         default => 'seabios',
 635     },
 636     vmgenid => {
 637         type => 'string',
 638         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 639         format_description => 'UUID',
 640         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 641             ." to disable explicitly.",
 642         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 643             ." value identifier to the guest OS. This allows to notify the guest operating system"
 644             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 645             ." execution or creation from a template). The guest operating system notices the"
 646             ." change, and is then able to react as appropriate by marking its copies of"
 647             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 648             ."Note that auto-creation only works when done through API/CLI create or update methods"
 649             .", but not when manually editing the config file.",
 650         default => "1 (autogenerated)",
 651         optional => 1,
 652     },
 653     hookscript => {
 654         type => 'string',
 655         format => 'pve-volume-id',
 656         optional => 1,
 657         description => "Script that will be executed during various steps in the vms lifetime.",
 658     },
 659     ivshmem => {
 660         type => 'string',
 661         format => $ivshmem_fmt,
 662         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 663             ." the host.",
 664         optional => 1,
 665     },
 666     audio0 => {
 667         type => 'string',
 668         format => $audio_fmt,
 669         description => "Configure a audio device, useful in combination with QXL/Spice.",
 670         optional => 1
 671     },
 672     spice_enhancements => {
 673         type => 'string',
 674         format => $spice_enhancements_fmt,
 675         description => "Configure additional enhancements for SPICE.",
 676         optional => 1
 677     },
 678     tags => {
 679         type => 'string', format => 'pve-tag-list',
 680         description => 'Tags of the VM. This is only meta information.',
 681         optional => 1,
 682     },
 683     rng0 => {
 684         type => 'string',
 685         format => $rng_fmt,
 686         description => "Configure a VirtIO-based Random Number Generator.",
 687         optional => 1,
 688     },
 689 };
 690
 691 my $cicustom_fmt = {
 692     meta => {
 693         type => 'string',
 694         optional => 1,
 695         description => 'Specify a custom file containing all meta data passed to the VM via"
 696             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 697         format => 'pve-volume-id',
 698         format_description => 'volume',
 699     },
 700     network => {
 701         type => 'string',
 702         optional => 1,
 703         description => 'Specify a custom file containing all network data passed to the VM via'
 704             .' cloud-init.',
 705         format => 'pve-volume-id',
 706         format_description => 'volume',
 707     },
 708     user => {
 709         type => 'string',
 710         optional => 1,
 711         description => 'Specify a custom file containing all user data passed to the VM via'
 712             .' cloud-init.',
 713         format => 'pve-volume-id',
 714         format_description => 'volume',
 715     },
 716 };
 717 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 718
 719 my $confdesc_cloudinit = {
 720     citype => {
 721         optional => 1,
 722         type => 'string',
 723         description => 'Specifies the cloud-init configuration format. The default depends on the'
 724             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 725             .' and `configdrive2` for windows.',
 726         enum => ['configdrive2', 'nocloud'],
 727     },
 728     ciuser => {
 729         optional => 1,
 730         type => 'string',
 731         description => "cloud-init: User name to change ssh keys and password for instead of the"
 732             ." image's configured default user.",
 733     },
 734     cipassword => {
 735         optional => 1,
 736         type => 'string',
 737         description => 'cloud-init: Password to assign the user. Using this is generally not'
 738             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 739             .' support hashed passwords.',
 740     },
 741     cicustom => {
 742         optional => 1,
 743         type => 'string',
 744         description => 'cloud-init: Specify custom files to replace the automatically generated'
 745             .' ones at start.',
 746         format => 'pve-qm-cicustom',
 747     },
 748     searchdomain => {
 749         optional => 1,
 750         type => 'string',
 751         description => "cloud-init: Sets DNS search domains for a container. Create will'
 752             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 753             .' are set.",
 754     },
 755     nameserver => {
 756         optional => 1,
 757         type => 'string', format => 'address-list',
 758         description => "cloud-init: Sets DNS server IP address for a container. Create will'
 759             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 760             .' are set.",
 761     },
 762     sshkeys => {
 763         optional => 1,
 764         type => 'string',
 765         format => 'urlencoded',
 766         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 767     },
 768 };
 769
 770 # what about other qemu settings ?
 771 #cpu => 'string',
 772 #machine => 'string',
 773 #fda => 'file',
 774 #fdb => 'file',
 775 #mtdblock => 'file',
 776 #sd => 'file',
 777 #pflash => 'file',
 778 #snapshot => 'bool',
 779 #bootp => 'file',
 780 ##tftp => 'dir',
 781 ##smb => 'dir',
 782 #kernel => 'file',
 783 #append => 'string',
 784 #initrd => 'file',
 785 ##soundhw => 'string',
 786
 787 while (my ($k, $v) = each %$confdesc) {
 788     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 789 }
 790
 791 my $MAX_USB_DEVICES = 5;
 792 my $MAX_NETS = 32;
 793 my $MAX_SERIAL_PORTS = 4;
 794 my $MAX_PARALLEL_PORTS = 3;
 795 my $MAX_NUMA = 8;
 796
 797 my $numa_fmt = {
 798     cpus => {
 799         type => "string",
 800         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 801         description => "CPUs accessing this NUMA node.",
 802         format_description => "id[-id];...",
 803     },
 804     memory => {
 805         type => "number",
 806         description => "Amount of memory this NUMA node provides.",
 807         optional => 1,
 808     },
 809     hostnodes => {
 810         type => "string",
 811         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 812         description => "Host NUMA nodes to use.",
 813         format_description => "id[-id];...",
 814         optional => 1,
 815     },
 816     policy => {
 817         type => 'string',
 818         enum => [qw(preferred bind interleave)],
 819         description => "NUMA allocation policy.",
 820         optional => 1,
 821     },
 822 };
 823 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 824 my $numadesc = {
 825     optional => 1,
 826     type => 'string', format => $numa_fmt,
 827     description => "NUMA topology.",
 828 };
 829 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 830
 831 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 832     $confdesc->{"numa$i"} = $numadesc;
 833 }
 834
 835 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 836                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 837                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 838 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 839
 840 my $net_fmt_bridge_descr = <<__EOD__;
 841 Bridge to attach the network device to. The Proxmox VE standard bridge
 842 is called 'vmbr0'.
 843
 844 If you do not specify a bridge, we create a kvm user (NATed) network
 845 device, which provides DHCP and DNS services. The following addresses
 846 are used:
 847
 848  10.0.2.2   Gateway
 849  10.0.2.3   DNS Server
 850  10.0.2.4   SMB Server
 851
 852 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 853 __EOD__
 854
 855 my $net_fmt = {
 856     macaddr  => get_standard_option('mac-addr', {
 857         description => "MAC address. That address must be unique withing your network. This is"
 858             ." automatically generated if not specified.",
 859     }),
 860     model => {
 861         type => 'string',
 862         description => "Network Card Model. The 'virtio' model provides the best performance with"
 863             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 864             ." best to use 'e1000'.",
 865         enum => $nic_model_list,
 866         default_key => 1,
 867     },
 868     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 869     bridge => {
 870         type => 'string',
 871         description => $net_fmt_bridge_descr,
 872         format_description => 'bridge',
 873         pattern => '[-_.\w\d]+',
 874         optional => 1,
 875     },
 876     queues => {
 877         type => 'integer',
 878         minimum => 0, maximum => 16,
 879         description => 'Number of packet queues to be used on the device.',
 880         optional => 1,
 881     },
 882     rate => {
 883         type => 'number',
 884         minimum => 0,
 885         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 886         optional => 1,
 887     },
 888     tag => {
 889         type => 'integer',
 890         minimum => 1, maximum => 4094,
 891         description => 'VLAN tag to apply to packets on this interface.',
 892         optional => 1,
 893     },
 894     trunks => {
 895         type => 'string',
 896         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 897         description => 'VLAN trunks to pass through this interface.',
 898         format_description => 'vlanid[;vlanid...]',
 899         optional => 1,
 900     },
 901     firewall => {
 902         type => 'boolean',
 903         description => 'Whether this interface should be protected by the firewall.',
 904         optional => 1,
 905     },
 906     link_down => {
 907         type => 'boolean',
 908         description => 'Whether this interface should be disconnected (like pulling the plug).',
 909         optional => 1,
 910     },
 911     mtu => {
 912         type => 'integer',
 913         minimum => 1, maximum => 65520,
 914         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 915         optional => 1,
 916     },
 917 };
 918
 919 my $netdesc = {
 920     optional => 1,
 921     type => 'string', format => $net_fmt,
 922     description => "Specify network devices.",
 923 };
 924
 925 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 926
 927 my $ipconfig_fmt = {
 928     ip => {
 929         type => 'string',
 930         format => 'pve-ipv4-config',
 931         format_description => 'IPv4Format/CIDR',
 932         description => 'IPv4 address in CIDR format.',
 933         optional => 1,
 934         default => 'dhcp',
 935     },
 936     gw => {
 937         type => 'string',
 938         format => 'ipv4',
 939         format_description => 'GatewayIPv4',
 940         description => 'Default gateway for IPv4 traffic.',
 941         optional => 1,
 942         requires => 'ip',
 943     },
 944     ip6 => {
 945         type => 'string',
 946         format => 'pve-ipv6-config',
 947         format_description => 'IPv6Format/CIDR',
 948         description => 'IPv6 address in CIDR format.',
 949         optional => 1,
 950         default => 'dhcp',
 951     },
 952     gw6 => {
 953         type => 'string',
 954         format => 'ipv6',
 955         format_description => 'GatewayIPv6',
 956         description => 'Default gateway for IPv6 traffic.',
 957         optional => 1,
 958         requires => 'ip6',
 959     },
 960 };
 961 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 962 my $ipconfigdesc = {
 963     optional => 1,
 964     type => 'string', format => 'pve-qm-ipconfig',
 965     description => <<'EODESCR',
 966 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 967
 968 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 969
 970 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
 971 gateway should be provided.
 972 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 973
 974 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
 975 dhcp on IPv4.
 976 EODESCR
 977 };
 978 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 979
 980 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 981     $confdesc->{"net$i"} = $netdesc;
 982     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 983 }
 984
 985 foreach my $key (keys %$confdesc_cloudinit) {
 986     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 987 }
 988
 989 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 990 sub verify_volume_id_or_qm_path {
 991     my ($volid, $noerr) = @_;
 992
 993     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 994         return $volid;
 995     }
 996
 997     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 998     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 999     if ($@) {
1000         return if $noerr;
1001         die $@;
1002     }
1003     return $volid;
1004 }
1005
1006 my $usb_fmt = {
1007     host => {
1008         default_key => 1,
1009         type => 'string', format => 'pve-qm-usb-device',
1010         format_description => 'HOSTUSBDEVICE|spice',
1011         description => <<EODESCR,
1012 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1013
1014  'bus-port(.port)*' (decimal numbers) or
1015  'vendor_id:product_id' (hexadeciaml numbers) or
1016  'spice'
1017
1018 You can use the 'lsusb -t' command to list existing usb devices.
1019
1020 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1021 machines - use with special care.
1022
1023 The value 'spice' can be used to add a usb redirection devices for spice.
1024 EODESCR
1025     },
1026     usb3 => {
1027         optional => 1,
1028         type => 'boolean',
1029         description => "Specifies whether if given host option is a USB3 device or port.",
1030         default => 0,
1031     },
1032 };
1033
1034 my $usbdesc = {
1035     optional => 1,
1036     type => 'string', format => $usb_fmt,
1037     description => "Configure an USB device (n is 0 to 4).",
1038 };
1039 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1040
1041 my $serialdesc = {
1042         optional => 1,
1043         type => 'string',
1044         pattern => '(/dev/.+|socket)',
1045         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1046         verbose_description =>  <<EODESCR,
1047 Create a serial device inside the VM (n is 0 to 3), and pass through a
1048 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1049 host side (use 'qm terminal' to open a terminal connection).
1050
1051 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1052 use with special care.
1053
1054 CAUTION: Experimental! User reported problems with this option.
1055 EODESCR
1056 };
1057
1058 my $paralleldesc= {
1059         optional => 1,
1060         type => 'string',
1061         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1062         description =>  "Map host parallel devices (n is 0 to 2).",
1063         verbose_description =>  <<EODESCR,
1064 Map host parallel devices (n is 0 to 2).
1065
1066 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1067 machines - use with special care.
1068
1069 CAUTION: Experimental! User reported problems with this option.
1070 EODESCR
1071 };
1072
1073 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1074     $confdesc->{"parallel$i"} = $paralleldesc;
1075 }
1076
1077 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1078     $confdesc->{"serial$i"} = $serialdesc;
1079 }
1080
1081 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1082     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1083 }
1084
1085 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1086     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1087 }
1088
1089 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1090     $confdesc->{"usb$i"} = $usbdesc;
1091 }
1092
1093 my $boot_fmt = {
1094     legacy => {
1095         optional => 1,
1096         default_key => 1,
1097         type => 'string',
1098         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1099                      . " Deprecated, use 'order=' instead.",
1100         pattern => '[acdn]{1,4}',
1101         format_description => "[acdn]{1,4}",
1102
1103         # note: this is also the fallback if boot: is not given at all
1104         default => 'cdn',
1105     },
1106     order => {
1107         optional => 1,
1108         type => 'string',
1109         format => 'pve-qm-bootdev-list',
1110         format_description => "device[;device...]",
1111         description => <<EODESC,
1112 The guest will attempt to boot from devices in the order they appear here.
1113
1114 Disks, optical drives and passed-through storage USB devices will be directly
1115 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1116 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1117
1118 Note that only devices in this list will be marked as bootable and thus loaded
1119 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1120 (e.g. software-raid), you need to specify all of them here.
1121
1122 Overrides the deprecated 'legacy=[acdn]*' value when given.
1123 EODESC
1124     },
1125 };
1126 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1127
1128 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1129 sub verify_bootdev {
1130     my ($dev, $noerr) = @_;
1131
1132     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && $dev !~ m/^efidisk/;
1133
1134     my $check = sub {
1135         my ($base) = @_;
1136         return 0 if $dev !~ m/^$base\d+$/;
1137         return 0 if !$confdesc->{$dev};
1138         return 1;
1139     };
1140
1141     return $dev if $check->("net");
1142     return $dev if $check->("usb");
1143     return $dev if $check->("hostpci");
1144
1145     return if $noerr;
1146     die "invalid boot device '$dev'\n";
1147 }
1148
1149 sub print_bootorder {
1150     my ($devs) = @_;
1151     my $data = { order => join(';', @$devs) };
1152     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1153 }
1154
1155 my $kvm_api_version = 0;
1156
1157 sub kvm_version {
1158     return $kvm_api_version if $kvm_api_version;
1159
1160     open my $fh, '<', '/dev/kvm'
1161         or return;
1162
1163     # 0xae00 => KVM_GET_API_VERSION
1164     $kvm_api_version = ioctl($fh, 0xae00, 0);
1165
1166     return $kvm_api_version;
1167 }
1168
1169 my $kvm_user_version = {};
1170 my $kvm_mtime = {};
1171
1172 sub kvm_user_version {
1173     my ($binary) = @_;
1174
1175     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1176     my $st = stat($binary);
1177
1178     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1179     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1180         $cachedmtime == $st->mtime;
1181
1182     $kvm_user_version->{$binary} = 'unknown';
1183     $kvm_mtime->{$binary} = $st->mtime;
1184
1185     my $code = sub {
1186         my $line = shift;
1187         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1188             $kvm_user_version->{$binary} = $2;
1189         }
1190     };
1191
1192     eval { run_command([$binary, '--version'], outfunc => $code); };
1193     warn $@ if $@;
1194
1195     return $kvm_user_version->{$binary};
1196
1197 }
1198 my sub extract_version {
1199     my ($machine_type, $version) = @_;
1200     $version = kvm_user_version() if !defined($version);
1201     PVE::QemuServer::Machine::extract_version($machine_type, $version)
1202 }
1203
1204 sub kernel_has_vhost_net {
1205     return -c '/dev/vhost-net';
1206 }
1207
1208 sub option_exists {
1209     my $key = shift;
1210     return defined($confdesc->{$key});
1211 }
1212
1213 my $cdrom_path;
1214 sub get_cdrom_path {
1215
1216     return  $cdrom_path if $cdrom_path;
1217
1218     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1219     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1220     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1221 }
1222
1223 sub get_iso_path {
1224     my ($storecfg, $vmid, $cdrom) = @_;
1225
1226     if ($cdrom eq 'cdrom') {
1227         return get_cdrom_path();
1228     } elsif ($cdrom eq 'none') {
1229         return '';
1230     } elsif ($cdrom =~ m|^/|) {
1231         return $cdrom;
1232     } else {
1233         return PVE::Storage::path($storecfg, $cdrom);
1234     }
1235 }
1236
1237 # try to convert old style file names to volume IDs
1238 sub filename_to_volume_id {
1239     my ($vmid, $file, $media) = @_;
1240
1241      if (!($file eq 'none' || $file eq 'cdrom' ||
1242           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1243
1244         return if $file =~ m|/|;
1245
1246         if ($media && $media eq 'cdrom') {
1247             $file = "local:iso/$file";
1248         } else {
1249             $file = "local:$vmid/$file";
1250         }
1251     }
1252
1253     return $file;
1254 }
1255
1256 sub verify_media_type {
1257     my ($opt, $vtype, $media) = @_;
1258
1259     return if !$media;
1260
1261     my $etype;
1262     if ($media eq 'disk') {
1263         $etype = 'images';
1264     } elsif ($media eq 'cdrom') {
1265         $etype = 'iso';
1266     } else {
1267         die "internal error";
1268     }
1269
1270     return if ($vtype eq $etype);
1271
1272     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1273 }
1274
1275 sub cleanup_drive_path {
1276     my ($opt, $storecfg, $drive) = @_;
1277
1278     # try to convert filesystem paths to volume IDs
1279
1280     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1281         ($drive->{file} !~ m|^/dev/.+|) &&
1282         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1283         ($drive->{file} !~ m/^\d+$/)) {
1284         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1285         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1286             if !$vtype;
1287         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1288         verify_media_type($opt, $vtype, $drive->{media});
1289         $drive->{file} = $volid;
1290     }
1291
1292     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1293 }
1294
1295 sub parse_hotplug_features {
1296     my ($data) = @_;
1297
1298     my $res = {};
1299
1300     return $res if $data eq '0';
1301
1302     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1303
1304     foreach my $feature (PVE::Tools::split_list($data)) {
1305         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1306             $res->{$1} = 1;
1307         } else {
1308             die "invalid hotplug feature '$feature'\n";
1309         }
1310     }
1311     return $res;
1312 }
1313
1314 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1315 sub pve_verify_hotplug_features {
1316     my ($value, $noerr) = @_;
1317
1318     return $value if parse_hotplug_features($value);
1319
1320     return if $noerr;
1321
1322     die "unable to parse hotplug option\n";
1323 }
1324
1325 sub scsi_inquiry {
1326     my($fh, $noerr) = @_;
1327
1328     my $SG_IO = 0x2285;
1329     my $SG_GET_VERSION_NUM = 0x2282;
1330
1331     my $versionbuf = "\x00" x 8;
1332     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1333     if (!$ret) {
1334         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1335         return;
1336     }
1337     my $version = unpack("I", $versionbuf);
1338     if ($version < 30000) {
1339         die "scsi generic interface too old\n"  if !$noerr;
1340         return;
1341     }
1342
1343     my $buf = "\x00" x 36;
1344     my $sensebuf = "\x00" x 8;
1345     my $cmd = pack("C x3 C x1", 0x12, 36);
1346
1347     # see /usr/include/scsi/sg.h
1348     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1349
1350     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1351                       length($sensebuf), 0, length($buf), $buf,
1352                       $cmd, $sensebuf, 6000);
1353
1354     $ret = ioctl($fh, $SG_IO, $packet);
1355     if (!$ret) {
1356         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1357         return;
1358     }
1359
1360     my @res = unpack($sg_io_hdr_t, $packet);
1361     if ($res[17] || $res[18]) {
1362         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1363         return;
1364     }
1365
1366     my $res = {};
1367     (my $byte0, my $byte1, $res->{vendor},
1368      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1369
1370     $res->{removable} = $byte1 & 128 ? 1 : 0;
1371     $res->{type} = $byte0 & 31;
1372
1373     return $res;
1374 }
1375
1376 sub path_is_scsi {
1377     my ($path) = @_;
1378
1379     my $fh = IO::File->new("+<$path") || return;
1380     my $res = scsi_inquiry($fh, 1);
1381     close($fh);
1382
1383     return $res;
1384 }
1385
1386 sub print_tabletdevice_full {
1387     my ($conf, $arch) = @_;
1388
1389     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1390
1391     # we use uhci for old VMs because tablet driver was buggy in older qemu
1392     my $usbbus;
1393     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1394         $usbbus = 'ehci';
1395     } else {
1396         $usbbus = 'uhci';
1397     }
1398
1399     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1400 }
1401
1402 sub print_keyboarddevice_full {
1403     my ($conf, $arch, $machine) = @_;
1404
1405     return if $arch ne 'aarch64';
1406
1407     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1408 }
1409
1410 sub print_drivedevice_full {
1411     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1412
1413     my $device = '';
1414     my $maxdev = 0;
1415
1416     my $drive_id = "$drive->{interface}$drive->{index}";
1417     if ($drive->{interface} eq 'virtio') {
1418         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1419         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1420         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1421     } elsif ($drive->{interface} eq 'scsi') {
1422
1423         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1424         my $unit = $drive->{index} % $maxdev;
1425         my $devicetype = 'hd';
1426         my $path = '';
1427         if (drive_is_cdrom($drive)) {
1428             $devicetype = 'cd';
1429         } else {
1430             if ($drive->{file} =~ m|^/|) {
1431                 $path = $drive->{file};
1432                 if (my $info = path_is_scsi($path)) {
1433                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1434                         $devicetype = 'block';
1435                     } elsif ($info->{type} == 1) { # tape
1436                         $devicetype = 'generic';
1437                     }
1438                 }
1439             } else {
1440                  $path = PVE::Storage::path($storecfg, $drive->{file});
1441             }
1442
1443             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1444             my $version = extract_version($machine_type, kvm_user_version());
1445             if ($path =~ m/^iscsi\:\/\// &&
1446                !min_version($version, 4, 1)) {
1447                 $devicetype = 'generic';
1448             }
1449         }
1450
1451         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1452             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1453         } else {
1454             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1455                 .",lun=$drive->{index}";
1456         }
1457         $device .= ",drive=drive-$drive_id,id=$drive_id";
1458
1459         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1460             $device .= ",rotation_rate=1";
1461         }
1462         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1463
1464     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1465         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1466         my $controller = int($drive->{index} / $maxdev);
1467         my $unit = $drive->{index} % $maxdev;
1468         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1469
1470         $device = "ide-$devicetype";
1471         if ($drive->{interface} eq 'ide') {
1472             $device .= ",bus=ide.$controller,unit=$unit";
1473         } else {
1474             $device .= ",bus=ahci$controller.$unit";
1475         }
1476         $device .= ",drive=drive-$drive_id,id=$drive_id";
1477
1478         if ($devicetype eq 'hd') {
1479             if (my $model = $drive->{model}) {
1480                 $model = URI::Escape::uri_unescape($model);
1481                 $device .= ",model=$model";
1482             }
1483             if ($drive->{ssd}) {
1484                 $device .= ",rotation_rate=1";
1485             }
1486         }
1487         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1488     } elsif ($drive->{interface} eq 'usb') {
1489         die "implement me";
1490         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1491     } else {
1492         die "unsupported interface type";
1493     }
1494
1495     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1496
1497     if (my $serial = $drive->{serial}) {
1498         $serial = URI::Escape::uri_unescape($serial);
1499         $device .= ",serial=$serial";
1500     }
1501
1502
1503     return $device;
1504 }
1505
1506 sub get_initiator_name {
1507     my $initiator;
1508
1509     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1510     while (defined(my $line = <$fh>)) {
1511         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1512         $initiator = $1;
1513         last;
1514     }
1515     $fh->close();
1516
1517     return $initiator;
1518 }
1519
1520 sub print_drive_commandline_full {
1521     my ($storecfg, $vmid, $drive) = @_;
1522
1523     my $path;
1524     my $volid = $drive->{file};
1525     my $format;
1526
1527     if (drive_is_cdrom($drive)) {
1528         $path = get_iso_path($storecfg, $vmid, $volid);
1529     } else {
1530         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1531         if ($storeid) {
1532             $path = PVE::Storage::path($storecfg, $volid);
1533             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1534             $format = qemu_img_format($scfg, $volname);
1535         } else {
1536             $path = $volid;
1537             $format = "raw";
1538         }
1539    }
1540
1541     my $opts = '';
1542     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1543     foreach my $o (@qemu_drive_options) {
1544         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1545     }
1546
1547     # snapshot only accepts on|off
1548     if (defined($drive->{snapshot})) {
1549         my $v = $drive->{snapshot} ? 'on' : 'off';
1550         $opts .= ",snapshot=$v";
1551     }
1552
1553     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1554         my ($dir, $qmpname) = @$type;
1555         if (my $v = $drive->{"mbps$dir"}) {
1556             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1557         }
1558         if (my $v = $drive->{"mbps${dir}_max"}) {
1559             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1560         }
1561         if (my $v = $drive->{"bps${dir}_max_length"}) {
1562             $opts .= ",throttling.bps$qmpname-max-length=$v";
1563         }
1564         if (my $v = $drive->{"iops${dir}"}) {
1565             $opts .= ",throttling.iops$qmpname=$v";
1566         }
1567         if (my $v = $drive->{"iops${dir}_max"}) {
1568             $opts .= ",throttling.iops$qmpname-max=$v";
1569         }
1570         if (my $v = $drive->{"iops${dir}_max_length"}) {
1571             $opts .= ",throttling.iops$qmpname-max-length=$v";
1572         }
1573     }
1574
1575     $opts .= ",format=$format" if $format && !$drive->{format};
1576
1577     my $cache_direct = 0;
1578
1579     if (my $cache = $drive->{cache}) {
1580         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1581     } elsif (!drive_is_cdrom($drive)) {
1582         $opts .= ",cache=none";
1583         $cache_direct = 1;
1584     }
1585
1586     # aio native works only with O_DIRECT
1587     if (!$drive->{aio}) {
1588         if($cache_direct) {
1589             $opts .= ",aio=native";
1590         } else {
1591             $opts .= ",aio=threads";
1592         }
1593     }
1594
1595     if (!drive_is_cdrom($drive)) {
1596         my $detectzeroes;
1597         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1598             $detectzeroes = 'off';
1599         } elsif ($drive->{discard}) {
1600             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1601         } else {
1602             # This used to be our default with discard not being specified:
1603             $detectzeroes = 'on';
1604         }
1605         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1606     }
1607
1608     my $pathinfo = $path ? "file=$path," : '';
1609
1610     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1611 }
1612
1613 sub print_netdevice_full {
1614     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1615
1616     my $device = $net->{model};
1617     if ($net->{model} eq 'virtio') {
1618          $device = 'virtio-net-pci';
1619      };
1620
1621     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1622     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1623     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1624         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1625         # and out of each queue plus one config interrupt and control vector queue
1626         my $vectors = $net->{queues} * 2 + 2;
1627         $tmpstr .= ",vectors=$vectors,mq=on";
1628     }
1629     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1630
1631     if (my $mtu = $net->{mtu}) {
1632         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1633             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1634             if ($mtu == 1) {
1635                  $mtu = $bridge_mtu;
1636             } elsif ($mtu < 576) {
1637                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1638             } elsif ($mtu > $bridge_mtu) {
1639                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1640             }
1641             $tmpstr .= ",host_mtu=$mtu";
1642         } else {
1643             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1644         }
1645     }
1646
1647     if ($use_old_bios_files) {
1648         my $romfile;
1649         if ($device eq 'virtio-net-pci') {
1650             $romfile = 'pxe-virtio.rom';
1651         } elsif ($device eq 'e1000') {
1652             $romfile = 'pxe-e1000.rom';
1653         } elsif ($device eq 'ne2k') {
1654             $romfile = 'pxe-ne2k_pci.rom';
1655         } elsif ($device eq 'pcnet') {
1656             $romfile = 'pxe-pcnet.rom';
1657         } elsif ($device eq 'rtl8139') {
1658             $romfile = 'pxe-rtl8139.rom';
1659         }
1660         $tmpstr .= ",romfile=$romfile" if $romfile;
1661     }
1662
1663     return $tmpstr;
1664 }
1665
1666 sub print_netdev_full {
1667     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1668
1669     my $i = '';
1670     if ($netid =~ m/^net(\d+)$/) {
1671         $i = int($1);
1672     }
1673
1674     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1675
1676     my $ifname = "tap${vmid}i$i";
1677
1678     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1679     die "interface name '$ifname' is too long (max 15 character)\n"
1680         if length($ifname) >= 16;
1681
1682     my $vhostparam = '';
1683     if (is_native($arch)) {
1684         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1685     }
1686
1687     my $vmname = $conf->{name} || "vm$vmid";
1688
1689     my $netdev = "";
1690     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1691
1692     if ($net->{bridge}) {
1693         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1694             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1695     } else {
1696         $netdev = "type=user,id=$netid,hostname=$vmname";
1697     }
1698
1699     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1700
1701     return $netdev;
1702 }
1703
1704 my $vga_map = {
1705     'cirrus' => 'cirrus-vga',
1706     'std' => 'VGA',
1707     'vmware' => 'vmware-svga',
1708     'virtio' => 'virtio-vga',
1709 };
1710
1711 sub print_vga_device {
1712     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1713
1714     my $type = $vga_map->{$vga->{type}};
1715     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1716         $type = 'virtio-gpu';
1717     }
1718     my $vgamem_mb = $vga->{memory};
1719
1720     my $max_outputs = '';
1721     if ($qxlnum) {
1722         $type = $id ? 'qxl' : 'qxl-vga';
1723
1724         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1725             # set max outputs so linux can have up to 4 qxl displays with one device
1726             if (min_version($machine_version, 4, 1)) {
1727                 $max_outputs = ",max_outputs=4";
1728             }
1729         }
1730     }
1731
1732     die "no devicetype for $vga->{type}\n" if !$type;
1733
1734     my $memory = "";
1735     if ($vgamem_mb) {
1736         if ($vga->{type} eq 'virtio') {
1737             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1738             $memory = ",max_hostmem=$bytes";
1739         } elsif ($qxlnum) {
1740             # from https://www.spice-space.org/multiple-monitors.html
1741             $memory = ",vgamem_mb=$vga->{memory}";
1742             my $ram = $vgamem_mb * 4;
1743             my $vram = $vgamem_mb * 2;
1744             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1745         } else {
1746             $memory = ",vgamem_mb=$vga->{memory}";
1747         }
1748     } elsif ($qxlnum && $id) {
1749         $memory = ",ram_size=67108864,vram_size=33554432";
1750     }
1751
1752     my $edidoff = "";
1753     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1754         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1755     }
1756
1757     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1758     my $vgaid = "vga" . ($id // '');
1759     my $pciaddr;
1760
1761     if ($q35 && $vgaid eq 'vga') {
1762         # the first display uses pcie.0 bus on q35 machines
1763         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1764     } else {
1765         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1766     }
1767
1768     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1769 }
1770
1771 sub parse_number_sets {
1772     my ($set) = @_;
1773     my $res = [];
1774     foreach my $part (split(/;/, $set)) {
1775         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1776             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1777             push @$res, [ $1, $2 ];
1778         } else {
1779             die "invalid range: $part\n";
1780         }
1781     }
1782     return $res;
1783 }
1784
1785 sub parse_numa {
1786     my ($data) = @_;
1787
1788     my $res = parse_property_string($numa_fmt, $data);
1789     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1790     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1791     return $res;
1792 }
1793
1794 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1795 sub parse_net {
1796     my ($data) = @_;
1797
1798     my $res = eval { parse_property_string($net_fmt, $data) };
1799     if ($@) {
1800         warn $@;
1801         return;
1802     }
1803     if (!defined($res->{macaddr})) {
1804         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1805         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1806     }
1807     return $res;
1808 }
1809
1810 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1811 sub parse_ipconfig {
1812     my ($data) = @_;
1813
1814     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1815     if ($@) {
1816         warn $@;
1817         return;
1818     }
1819
1820     if ($res->{gw} && !$res->{ip}) {
1821         warn 'gateway specified without specifying an IP address';
1822         return;
1823     }
1824     if ($res->{gw6} && !$res->{ip6}) {
1825         warn 'IPv6 gateway specified without specifying an IPv6 address';
1826         return;
1827     }
1828     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1829         warn 'gateway specified together with DHCP';
1830         return;
1831     }
1832     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1833         # gw6 + auto/dhcp
1834         warn "IPv6 gateway specified together with $res->{ip6} address";
1835         return;
1836     }
1837
1838     if (!$res->{ip} && !$res->{ip6}) {
1839         return { ip => 'dhcp', ip6 => 'dhcp' };
1840     }
1841
1842     return $res;
1843 }
1844
1845 sub print_net {
1846     my $net = shift;
1847
1848     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1849 }
1850
1851 sub add_random_macs {
1852     my ($settings) = @_;
1853
1854     foreach my $opt (keys %$settings) {
1855         next if $opt !~ m/^net(\d+)$/;
1856         my $net = parse_net($settings->{$opt});
1857         next if !$net;
1858         $settings->{$opt} = print_net($net);
1859     }
1860 }
1861
1862 sub vm_is_volid_owner {
1863     my ($storecfg, $vmid, $volid) = @_;
1864
1865     if ($volid !~  m|^/|) {
1866         my ($path, $owner);
1867         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1868         if ($owner && ($owner == $vmid)) {
1869             return 1;
1870         }
1871     }
1872
1873     return;
1874 }
1875
1876 sub vmconfig_register_unused_drive {
1877     my ($storecfg, $vmid, $conf, $drive) = @_;
1878
1879     if (drive_is_cloudinit($drive)) {
1880         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1881         warn $@ if $@;
1882     } elsif (!drive_is_cdrom($drive)) {
1883         my $volid = $drive->{file};
1884         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1885             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1886         }
1887     }
1888 }
1889
1890 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1891 my $smbios1_fmt = {
1892     uuid => {
1893         type => 'string',
1894         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1895         format_description => 'UUID',
1896         description => "Set SMBIOS1 UUID.",
1897         optional => 1,
1898     },
1899     version => {
1900         type => 'string',
1901         pattern => '[A-Za-z0-9+\/]+={0,2}',
1902         format_description => 'Base64 encoded string',
1903         description => "Set SMBIOS1 version.",
1904         optional => 1,
1905     },
1906     serial => {
1907         type => 'string',
1908         pattern => '[A-Za-z0-9+\/]+={0,2}',
1909         format_description => 'Base64 encoded string',
1910         description => "Set SMBIOS1 serial number.",
1911         optional => 1,
1912     },
1913     manufacturer => {
1914         type => 'string',
1915         pattern => '[A-Za-z0-9+\/]+={0,2}',
1916         format_description => 'Base64 encoded string',
1917         description => "Set SMBIOS1 manufacturer.",
1918         optional => 1,
1919     },
1920     product => {
1921         type => 'string',
1922         pattern => '[A-Za-z0-9+\/]+={0,2}',
1923         format_description => 'Base64 encoded string',
1924         description => "Set SMBIOS1 product ID.",
1925         optional => 1,
1926     },
1927     sku => {
1928         type => 'string',
1929         pattern => '[A-Za-z0-9+\/]+={0,2}',
1930         format_description => 'Base64 encoded string',
1931         description => "Set SMBIOS1 SKU string.",
1932         optional => 1,
1933     },
1934     family => {
1935         type => 'string',
1936         pattern => '[A-Za-z0-9+\/]+={0,2}',
1937         format_description => 'Base64 encoded string',
1938         description => "Set SMBIOS1 family string.",
1939         optional => 1,
1940     },
1941     base64 => {
1942         type => 'boolean',
1943         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
1944         optional => 1,
1945     },
1946 };
1947
1948 sub parse_smbios1 {
1949     my ($data) = @_;
1950
1951     my $res = eval { parse_property_string($smbios1_fmt, $data) };
1952     warn $@ if $@;
1953     return $res;
1954 }
1955
1956 sub print_smbios1 {
1957     my ($smbios1) = @_;
1958     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1959 }
1960
1961 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1962
1963 sub parse_watchdog {
1964     my ($value) = @_;
1965
1966     return if !$value;
1967
1968     my $res = eval { parse_property_string($watchdog_fmt, $value) };
1969     warn $@ if $@;
1970     return $res;
1971 }
1972
1973 sub parse_guest_agent {
1974     my ($value) = @_;
1975
1976     return {} if !defined($value->{agent});
1977
1978     my $res = eval { parse_property_string($agent_fmt, $value->{agent}) };
1979     warn $@ if $@;
1980
1981     # if the agent is disabled ignore the other potentially set properties
1982     return {} if !$res->{enabled};
1983     return $res;
1984 }
1985
1986 sub parse_vga {
1987     my ($value) = @_;
1988
1989     return {} if !$value;
1990     my $res = eval { parse_property_string($vga_fmt, $value) };
1991     warn $@ if $@;
1992     return $res;
1993 }
1994
1995 sub parse_rng {
1996     my ($value) = @_;
1997
1998     return if !$value;
1999
2000     my $res = eval { parse_property_string($rng_fmt, $value) };
2001     warn $@ if $@;
2002     return $res;
2003 }
2004
2005 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2006 sub verify_usb_device {
2007     my ($value, $noerr) = @_;
2008
2009     return $value if parse_usb_device($value);
2010
2011     return if $noerr;
2012
2013     die "unable to parse usb device\n";
2014 }
2015
2016 # add JSON properties for create and set function
2017 sub json_config_properties {
2018     my $prop = shift;
2019
2020     foreach my $opt (keys %$confdesc) {
2021         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' ||
2022             $opt eq 'runningmachine' || $opt eq 'runningcpu';
2023         $prop->{$opt} = $confdesc->{$opt};
2024     }
2025
2026     return $prop;
2027 }
2028
2029 # return copy of $confdesc_cloudinit to generate documentation
2030 sub cloudinit_config_properties {
2031
2032     return dclone($confdesc_cloudinit);
2033 }
2034
2035 sub check_type {
2036     my ($key, $value) = @_;
2037
2038     die "unknown setting '$key'\n" if !$confdesc->{$key};
2039
2040     my $type = $confdesc->{$key}->{type};
2041
2042     if (!defined($value)) {
2043         die "got undefined value\n";
2044     }
2045
2046     if ($value =~ m/[\n\r]/) {
2047         die "property contains a line feed\n";
2048     }
2049
2050     if ($type eq 'boolean') {
2051         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2052         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2053         die "type check ('boolean') failed - got '$value'\n";
2054     } elsif ($type eq 'integer') {
2055         return int($1) if $value =~ m/^(\d+)$/;
2056         die "type check ('integer') failed - got '$value'\n";
2057     } elsif ($type eq 'number') {
2058         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2059         die "type check ('number') failed - got '$value'\n";
2060     } elsif ($type eq 'string') {
2061         if (my $fmt = $confdesc->{$key}->{format}) {
2062             PVE::JSONSchema::check_format($fmt, $value);
2063             return $value;
2064         }
2065         $value =~ s/^\"(.*)\"$/$1/;
2066         return $value;
2067     } else {
2068         die "internal error"
2069     }
2070 }
2071
2072 sub destroy_vm {
2073     my ($storecfg, $vmid, $skiplock, $replacement_conf) = @_;
2074
2075     my $conf = PVE::QemuConfig->load_config($vmid);
2076
2077     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2078
2079     if ($conf->{template}) {
2080         # check if any base image is still used by a linked clone
2081         PVE::QemuConfig->foreach_volume($conf, sub {
2082                 my ($ds, $drive) = @_;
2083                 return if drive_is_cdrom($drive);
2084
2085                 my $volid = $drive->{file};
2086                 return if !$volid || $volid =~ m|^/|;
2087
2088                 die "base volume '$volid' is still in use by linked cloned\n"
2089                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2090
2091         });
2092     }
2093
2094     # only remove disks owned by this VM
2095     PVE::QemuConfig->foreach_volume($conf, sub {
2096         my ($ds, $drive) = @_;
2097         return if drive_is_cdrom($drive, 1);
2098
2099         my $volid = $drive->{file};
2100         return if !$volid || $volid =~ m|^/|;
2101
2102         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2103         return if !$path || !$owner || ($owner != $vmid);
2104
2105         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2106         warn "Could not remove disk '$volid', check manually: $@" if $@;
2107     });
2108
2109     # also remove unused disk
2110     my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2111     PVE::Storage::foreach_volid($vmdisks, sub {
2112         my ($volid, $sid, $volname, $d) = @_;
2113         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2114         warn $@ if $@;
2115     });
2116
2117     if (defined $replacement_conf) {
2118         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2119     } else {
2120         PVE::QemuConfig->destroy_config($vmid);
2121     }
2122 }
2123
2124 sub parse_vm_config {
2125     my ($filename, $raw) = @_;
2126
2127     return if !defined($raw);
2128
2129     my $res = {
2130         digest => Digest::SHA::sha1_hex($raw),
2131         snapshots => {},
2132         pending => {},
2133     };
2134
2135     $filename =~ m|/qemu-server/(\d+)\.conf$|
2136         || die "got strange filename '$filename'";
2137
2138     my $vmid = $1;
2139
2140     my $conf = $res;
2141     my $descr;
2142     my $section = '';
2143
2144     my @lines = split(/\n/, $raw);
2145     foreach my $line (@lines) {
2146         next if $line =~ m/^\s*$/;
2147
2148         if ($line =~ m/^\[PENDING\]\s*$/i) {
2149             $section = 'pending';
2150             if (defined($descr)) {
2151                 $descr =~ s/\s+$//;
2152                 $conf->{description} = $descr;
2153             }
2154             $descr = undef;
2155             $conf = $res->{$section} = {};
2156             next;
2157
2158         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2159             $section = $1;
2160             if (defined($descr)) {
2161                 $descr =~ s/\s+$//;
2162                 $conf->{description} = $descr;
2163             }
2164             $descr = undef;
2165             $conf = $res->{snapshots}->{$section} = {};
2166             next;
2167         }
2168
2169         if ($line =~ m/^\#(.*)\s*$/) {
2170             $descr = '' if !defined($descr);
2171             $descr .= PVE::Tools::decode_text($1) . "\n";
2172             next;
2173         }
2174
2175         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2176             $descr = '' if !defined($descr);
2177             $descr .= PVE::Tools::decode_text($2);
2178         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2179             $conf->{snapstate} = $1;
2180         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2181             my $key = $1;
2182             my $value = $2;
2183             $conf->{$key} = $value;
2184         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2185             my $value = $1;
2186             if ($section eq 'pending') {
2187                 $conf->{delete} = $value; # we parse this later
2188             } else {
2189                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2190             }
2191         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2192             my $key = $1;
2193             my $value = $2;
2194             eval { $value = check_type($key, $value); };
2195             if ($@) {
2196                 warn "vm $vmid - unable to parse value of '$key' - $@";
2197             } else {
2198                 $key = 'ide2' if $key eq 'cdrom';
2199                 my $fmt = $confdesc->{$key}->{format};
2200                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2201                     my $v = parse_drive($key, $value);
2202                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2203                         $v->{file} = $volid;
2204                         $value = print_drive($v);
2205                     } else {
2206                         warn "vm $vmid - unable to parse value of '$key'\n";
2207                         next;
2208                     }
2209                 }
2210
2211                 $conf->{$key} = $value;
2212             }
2213         }
2214     }
2215
2216     if (defined($descr)) {
2217         $descr =~ s/\s+$//;
2218         $conf->{description} = $descr;
2219     }
2220     delete $res->{snapstate}; # just to be sure
2221
2222     return $res;
2223 }
2224
2225 sub write_vm_config {
2226     my ($filename, $conf) = @_;
2227
2228     delete $conf->{snapstate}; # just to be sure
2229
2230     if ($conf->{cdrom}) {
2231         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2232         $conf->{ide2} = $conf->{cdrom};
2233         delete $conf->{cdrom};
2234     }
2235
2236     # we do not use 'smp' any longer
2237     if ($conf->{sockets}) {
2238         delete $conf->{smp};
2239     } elsif ($conf->{smp}) {
2240         $conf->{sockets} = $conf->{smp};
2241         delete $conf->{cores};
2242         delete $conf->{smp};
2243     }
2244
2245     my $used_volids = {};
2246
2247     my $cleanup_config = sub {
2248         my ($cref, $pending, $snapname) = @_;
2249
2250         foreach my $key (keys %$cref) {
2251             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2252                 $key eq 'snapstate' || $key eq 'pending';
2253             my $value = $cref->{$key};
2254             if ($key eq 'delete') {
2255                 die "propertry 'delete' is only allowed in [PENDING]\n"
2256                     if !$pending;
2257                 # fixme: check syntax?
2258                 next;
2259             }
2260             eval { $value = check_type($key, $value); };
2261             die "unable to parse value of '$key' - $@" if $@;
2262
2263             $cref->{$key} = $value;
2264
2265             if (!$snapname && is_valid_drivename($key)) {
2266                 my $drive = parse_drive($key, $value);
2267                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2268             }
2269         }
2270     };
2271
2272     &$cleanup_config($conf);
2273
2274     &$cleanup_config($conf->{pending}, 1);
2275
2276     foreach my $snapname (keys %{$conf->{snapshots}}) {
2277         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2278         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2279     }
2280
2281     # remove 'unusedX' settings if we re-add a volume
2282     foreach my $key (keys %$conf) {
2283         my $value = $conf->{$key};
2284         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2285             delete $conf->{$key};
2286         }
2287     }
2288
2289     my $generate_raw_config = sub {
2290         my ($conf, $pending) = @_;
2291
2292         my $raw = '';
2293
2294         # add description as comment to top of file
2295         if (defined(my $descr = $conf->{description})) {
2296             if ($descr) {
2297                 foreach my $cl (split(/\n/, $descr)) {
2298                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2299                 }
2300             } else {
2301                 $raw .= "#\n" if $pending;
2302             }
2303         }
2304
2305         foreach my $key (sort keys %$conf) {
2306             next if $key =~ /^(digest|description|pending|snapshots)$/;
2307             $raw .= "$key: $conf->{$key}\n";
2308         }
2309         return $raw;
2310     };
2311
2312     my $raw = &$generate_raw_config($conf);
2313
2314     if (scalar(keys %{$conf->{pending}})){
2315         $raw .= "\n[PENDING]\n";
2316         $raw .= &$generate_raw_config($conf->{pending}, 1);
2317     }
2318
2319     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2320         $raw .= "\n[$snapname]\n";
2321         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2322     }
2323
2324     return $raw;
2325 }
2326
2327 sub load_defaults {
2328
2329     my $res = {};
2330
2331     # we use static defaults from our JSON schema configuration
2332     foreach my $key (keys %$confdesc) {
2333         if (defined(my $default = $confdesc->{$key}->{default})) {
2334             $res->{$key} = $default;
2335         }
2336     }
2337
2338     return $res;
2339 }
2340
2341 sub config_list {
2342     my $vmlist = PVE::Cluster::get_vmlist();
2343     my $res = {};
2344     return $res if !$vmlist || !$vmlist->{ids};
2345     my $ids = $vmlist->{ids};
2346     my $nodename = nodename();
2347
2348     foreach my $vmid (keys %$ids) {
2349         my $d = $ids->{$vmid};
2350         next if !$d->{node} || $d->{node} ne $nodename;
2351         next if !$d->{type} || $d->{type} ne 'qemu';
2352         $res->{$vmid}->{exists} = 1;
2353     }
2354     return $res;
2355 }
2356
2357 # test if VM uses local resources (to prevent migration)
2358 sub check_local_resources {
2359     my ($conf, $noerr) = @_;
2360
2361     my @loc_res = ();
2362
2363     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2364     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2365
2366     push @loc_res, "ivshmem" if $conf->{ivshmem};
2367
2368     foreach my $k (keys %$conf) {
2369         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2370         # sockets are safe: they will recreated be on the target side post-migrate
2371         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2372         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2373     }
2374
2375     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2376
2377     return \@loc_res;
2378 }
2379
2380 # check if used storages are available on all nodes (use by migrate)
2381 sub check_storage_availability {
2382     my ($storecfg, $conf, $node) = @_;
2383
2384     PVE::QemuConfig->foreach_volume($conf, sub {
2385         my ($ds, $drive) = @_;
2386
2387         my $volid = $drive->{file};
2388         return if !$volid;
2389
2390         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2391         return if !$sid;
2392
2393         # check if storage is available on both nodes
2394         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2395         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2396    });
2397 }
2398
2399 # list nodes where all VM images are available (used by has_feature API)
2400 sub shared_nodes {
2401     my ($conf, $storecfg) = @_;
2402
2403     my $nodelist = PVE::Cluster::get_nodelist();
2404     my $nodehash = { map { $_ => 1 } @$nodelist };
2405     my $nodename = nodename();
2406
2407     PVE::QemuConfig->foreach_volume($conf, sub {
2408         my ($ds, $drive) = @_;
2409
2410         my $volid = $drive->{file};
2411         return if !$volid;
2412
2413         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2414         if ($storeid) {
2415             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2416             if ($scfg->{disable}) {
2417                 $nodehash = {};
2418             } elsif (my $avail = $scfg->{nodes}) {
2419                 foreach my $node (keys %$nodehash) {
2420                     delete $nodehash->{$node} if !$avail->{$node};
2421                 }
2422             } elsif (!$scfg->{shared}) {
2423                 foreach my $node (keys %$nodehash) {
2424                     delete $nodehash->{$node} if $node ne $nodename
2425                 }
2426             }
2427         }
2428     });
2429
2430     return $nodehash
2431 }
2432
2433 sub check_local_storage_availability {
2434     my ($conf, $storecfg) = @_;
2435
2436     my $nodelist = PVE::Cluster::get_nodelist();
2437     my $nodehash = { map { $_ => {} } @$nodelist };
2438
2439     PVE::QemuConfig->foreach_volume($conf, sub {
2440         my ($ds, $drive) = @_;
2441
2442         my $volid = $drive->{file};
2443         return if !$volid;
2444
2445         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2446         if ($storeid) {
2447             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2448
2449             if ($scfg->{disable}) {
2450                 foreach my $node (keys %$nodehash) {
2451                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2452                 }
2453             } elsif (my $avail = $scfg->{nodes}) {
2454                 foreach my $node (keys %$nodehash) {
2455                     if (!$avail->{$node}) {
2456                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2457                     }
2458                 }
2459             }
2460         }
2461     });
2462
2463     foreach my $node (values %$nodehash) {
2464         if (my $unavail = $node->{unavailable_storages}) {
2465             $node->{unavailable_storages} = [ sort keys %$unavail ];
2466         }
2467     }
2468
2469     return $nodehash
2470 }
2471
2472 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2473 sub check_running {
2474     my ($vmid, $nocheck, $node) = @_;
2475
2476     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2477     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2478 }
2479
2480 sub vzlist {
2481
2482     my $vzlist = config_list();
2483
2484     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2485
2486     while (defined(my $de = $fd->read)) {
2487         next if $de !~ m/^(\d+)\.pid$/;
2488         my $vmid = $1;
2489         next if !defined($vzlist->{$vmid});
2490         if (my $pid = check_running($vmid)) {
2491             $vzlist->{$vmid}->{pid} = $pid;
2492         }
2493     }
2494
2495     return $vzlist;
2496 }
2497
2498 our $vmstatus_return_properties = {
2499     vmid => get_standard_option('pve-vmid'),
2500     status => {
2501         description => "Qemu process status.",
2502         type => 'string',
2503         enum => ['stopped', 'running'],
2504     },
2505     maxmem => {
2506         description => "Maximum memory in bytes.",
2507         type => 'integer',
2508         optional => 1,
2509         renderer => 'bytes',
2510     },
2511     maxdisk => {
2512         description => "Root disk size in bytes.",
2513         type => 'integer',
2514         optional => 1,
2515         renderer => 'bytes',
2516     },
2517     name => {
2518         description => "VM name.",
2519         type => 'string',
2520         optional => 1,
2521     },
2522     qmpstatus => {
2523         description => "Qemu QMP agent status.",
2524         type => 'string',
2525         optional => 1,
2526     },
2527     pid => {
2528         description => "PID of running qemu process.",
2529         type => 'integer',
2530         optional => 1,
2531     },
2532     uptime => {
2533         description => "Uptime.",
2534         type => 'integer',
2535         optional => 1,
2536         renderer => 'duration',
2537     },
2538     cpus => {
2539         description => "Maximum usable CPUs.",
2540         type => 'number',
2541         optional => 1,
2542     },
2543     lock => {
2544         description => "The current config lock, if any.",
2545         type => 'string',
2546         optional => 1,
2547     },
2548     tags => {
2549         description  => "The current configured tags, if any",
2550         type => 'string',
2551         optional => 1,
2552     },
2553 };
2554
2555 my $last_proc_pid_stat;
2556
2557 # get VM status information
2558 # This must be fast and should not block ($full == false)
2559 # We only query KVM using QMP if $full == true (this can be slow)
2560 sub vmstatus {
2561     my ($opt_vmid, $full) = @_;
2562
2563     my $res = {};
2564
2565     my $storecfg = PVE::Storage::config();
2566
2567     my $list = vzlist();
2568     my $defaults = load_defaults();
2569
2570     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2571
2572     my $cpucount = $cpuinfo->{cpus} || 1;
2573
2574     foreach my $vmid (keys %$list) {
2575         next if $opt_vmid && ($vmid ne $opt_vmid);
2576
2577         my $conf = PVE::QemuConfig->load_config($vmid);
2578
2579         my $d = { vmid => $vmid };
2580         $d->{pid} = $list->{$vmid}->{pid};
2581
2582         # fixme: better status?
2583         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2584
2585         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2586         if (defined($size)) {
2587             $d->{disk} = 0; # no info available
2588             $d->{maxdisk} = $size;
2589         } else {
2590             $d->{disk} = 0;
2591             $d->{maxdisk} = 0;
2592         }
2593
2594         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2595             * ($conf->{cores} || $defaults->{cores});
2596         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2597         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2598
2599         $d->{name} = $conf->{name} || "VM $vmid";
2600         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2601             : $defaults->{memory}*(1024*1024);
2602
2603         if ($conf->{balloon}) {
2604             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2605             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2606                 : $defaults->{shares};
2607         }
2608
2609         $d->{uptime} = 0;
2610         $d->{cpu} = 0;
2611         $d->{mem} = 0;
2612
2613         $d->{netout} = 0;
2614         $d->{netin} = 0;
2615
2616         $d->{diskread} = 0;
2617         $d->{diskwrite} = 0;
2618
2619         $d->{template} = PVE::QemuConfig->is_template($conf);
2620
2621         $d->{serial} = 1 if conf_has_serial($conf);
2622         $d->{lock} = $conf->{lock} if $conf->{lock};
2623         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2624
2625         $res->{$vmid} = $d;
2626     }
2627
2628     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2629     foreach my $dev (keys %$netdev) {
2630         next if $dev !~ m/^tap([1-9]\d*)i/;
2631         my $vmid = $1;
2632         my $d = $res->{$vmid};
2633         next if !$d;
2634
2635         $d->{netout} += $netdev->{$dev}->{receive};
2636         $d->{netin} += $netdev->{$dev}->{transmit};
2637
2638         if ($full) {
2639             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2640             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2641         }
2642
2643     }
2644
2645     my $ctime = gettimeofday;
2646
2647     foreach my $vmid (keys %$list) {
2648
2649         my $d = $res->{$vmid};
2650         my $pid = $d->{pid};
2651         next if !$pid;
2652
2653         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2654         next if !$pstat; # not running
2655
2656         my $used = $pstat->{utime} + $pstat->{stime};
2657
2658         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2659
2660         if ($pstat->{vsize}) {
2661             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2662         }
2663
2664         my $old = $last_proc_pid_stat->{$pid};
2665         if (!$old) {
2666             $last_proc_pid_stat->{$pid} = {
2667                 time => $ctime,
2668                 used => $used,
2669                 cpu => 0,
2670             };
2671             next;
2672         }
2673
2674         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2675
2676         if ($dtime > 1000) {
2677             my $dutime = $used -  $old->{used};
2678
2679             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2680             $last_proc_pid_stat->{$pid} = {
2681                 time => $ctime,
2682                 used => $used,
2683                 cpu => $d->{cpu},
2684             };
2685         } else {
2686             $d->{cpu} = $old->{cpu};
2687         }
2688     }
2689
2690     return $res if !$full;
2691
2692     my $qmpclient = PVE::QMPClient->new();
2693
2694     my $ballooncb = sub {
2695         my ($vmid, $resp) = @_;
2696
2697         my $info = $resp->{'return'};
2698         return if !$info->{max_mem};
2699
2700         my $d = $res->{$vmid};
2701
2702         # use memory assigned to VM
2703         $d->{maxmem} = $info->{max_mem};
2704         $d->{balloon} = $info->{actual};
2705
2706         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2707             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2708             $d->{freemem} = $info->{free_mem};
2709         }
2710
2711         $d->{ballooninfo} = $info;
2712     };
2713
2714     my $blockstatscb = sub {
2715         my ($vmid, $resp) = @_;
2716         my $data = $resp->{'return'} || [];
2717         my $totalrdbytes = 0;
2718         my $totalwrbytes = 0;
2719
2720         for my $blockstat (@$data) {
2721             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2722             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2723
2724             $blockstat->{device} =~ s/drive-//;
2725             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2726         }
2727         $res->{$vmid}->{diskread} = $totalrdbytes;
2728         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2729     };
2730
2731     my $statuscb = sub {
2732         my ($vmid, $resp) = @_;
2733
2734         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2735         # this fails if ballon driver is not loaded, so this must be
2736         # the last commnand (following command are aborted if this fails).
2737         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2738
2739         my $status = 'unknown';
2740         if (!defined($status = $resp->{'return'}->{status})) {
2741             warn "unable to get VM status\n";
2742             return;
2743         }
2744
2745         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2746     };
2747
2748     foreach my $vmid (keys %$list) {
2749         next if $opt_vmid && ($vmid ne $opt_vmid);
2750         next if !$res->{$vmid}->{pid}; # not running
2751         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2752     }
2753
2754     $qmpclient->queue_execute(undef, 2);
2755
2756     foreach my $vmid (keys %$list) {
2757         next if $opt_vmid && ($vmid ne $opt_vmid);
2758         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2759     }
2760
2761     return $res;
2762 }
2763
2764 sub conf_has_serial {
2765     my ($conf) = @_;
2766
2767     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2768         if ($conf->{"serial$i"}) {
2769             return 1;
2770         }
2771     }
2772
2773     return 0;
2774 }
2775
2776 sub conf_has_audio {
2777     my ($conf, $id) = @_;
2778
2779     $id //= 0;
2780     my $audio = $conf->{"audio$id"};
2781     return if !defined($audio);
2782
2783     my $audioproperties = parse_property_string($audio_fmt, $audio);
2784     my $audiodriver = $audioproperties->{driver} // 'spice';
2785
2786     return {
2787         dev => $audioproperties->{device},
2788         dev_id => "audiodev$id",
2789         backend => $audiodriver,
2790         backend_id => "$audiodriver-backend${id}",
2791     };
2792 }
2793
2794 sub audio_devs {
2795     my ($audio, $audiopciaddr, $machine_version) = @_;
2796
2797     my $devs = [];
2798
2799     my $id = $audio->{dev_id};
2800     my $audiodev = "";
2801     if (min_version($machine_version, 4, 2)) {
2802         $audiodev = ",audiodev=$audio->{backend_id}";
2803     }
2804
2805     if ($audio->{dev} eq 'AC97') {
2806         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
2807     } elsif ($audio->{dev} =~ /intel\-hda$/) {
2808         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
2809         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
2810         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
2811     } else {
2812         die "unkown audio device '$audio->{dev}', implement me!";
2813     }
2814
2815     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
2816
2817     return $devs;
2818 }
2819
2820 sub vga_conf_has_spice {
2821     my ($vga) = @_;
2822
2823     my $vgaconf = parse_vga($vga);
2824     my $vgatype = $vgaconf->{type};
2825     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
2826
2827     return $1 || 1;
2828 }
2829
2830 sub is_native($) {
2831     my ($arch) = @_;
2832     return get_host_arch() eq $arch;
2833 }
2834
2835 sub get_vm_arch {
2836     my ($conf) = @_;
2837     return $conf->{arch} // get_host_arch();
2838 }
2839
2840 my $default_machines = {
2841     x86_64 => 'pc',
2842     aarch64 => 'virt',
2843 };
2844
2845 sub get_vm_machine {
2846     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
2847
2848     my $machine = $forcemachine || $conf->{machine};
2849
2850     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
2851         $arch //= 'x86_64';
2852         $machine ||= $default_machines->{$arch};
2853         if ($add_pve_version) {
2854             $kvmversion //= kvm_user_version();
2855             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
2856             $machine .= "+pve$pvever";
2857         }
2858     }
2859
2860     if ($add_pve_version && $machine !~ m/\+pve\d+$/) {
2861         # for version-pinned machines that do not include a pve-version (e.g.
2862         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
2863         $machine .= '+pve0';
2864     }
2865
2866     return $machine;
2867 }
2868
2869 sub get_ovmf_files($) {
2870     my ($arch) = @_;
2871
2872     my $ovmf = $OVMF->{$arch}
2873         or die "no OVMF images known for architecture '$arch'\n";
2874
2875     return @$ovmf;
2876 }
2877
2878 my $Arch2Qemu = {
2879     aarch64 => '/usr/bin/qemu-system-aarch64',
2880     x86_64 => '/usr/bin/qemu-system-x86_64',
2881 };
2882 sub get_command_for_arch($) {
2883     my ($arch) = @_;
2884     return '/usr/bin/kvm' if is_native($arch);
2885
2886     my $cmd = $Arch2Qemu->{$arch}
2887         or die "don't know how to emulate architecture '$arch'\n";
2888     return $cmd;
2889 }
2890
2891 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
2892 # to use in a QEMU command line (-cpu element), first array_intersect the result
2893 # of query_supported_ with query_understood_. This is necessary because:
2894 #
2895 # a) query_understood_ returns flags the host cannot use and
2896 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
2897 #    flags, but CPU settings - with most of them being flags. Those settings
2898 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
2899 #
2900 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
2901 # expensive. If you need the value returned from this, you can get it much
2902 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
2903 # $accel being 'kvm' or 'tcg'.
2904 #
2905 # pvestatd calls this function on startup and whenever the QEMU/KVM version
2906 # changes, automatically populating pmxcfs.
2907 #
2908 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
2909 # since kvm and tcg machines support different flags
2910 #
2911 sub query_supported_cpu_flags {
2912     my ($arch) = @_;
2913
2914     $arch //= get_host_arch();
2915     my $default_machine = $default_machines->{$arch};
2916
2917     my $flags = {};
2918
2919     # FIXME: Once this is merged, the code below should work for ARM as well:
2920     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
2921     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
2922         $arch eq "aarch64";
2923
2924     my $kvm_supported = defined(kvm_version());
2925     my $qemu_cmd = get_command_for_arch($arch);
2926     my $fakevmid = -1;
2927     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
2928
2929     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
2930     my $query_supported_run_qemu = sub {
2931         my ($kvm) = @_;
2932
2933         my $flags = {};
2934         my $cmd = [
2935             $qemu_cmd,
2936             '-machine', $default_machine,
2937             '-display', 'none',
2938             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server,nowait",
2939             '-mon', 'chardev=qmp,mode=control',
2940             '-pidfile', $pidfile,
2941             '-S', '-daemonize'
2942         ];
2943
2944         if (!$kvm) {
2945             push @$cmd, '-accel', 'tcg';
2946         }
2947
2948         my $rc = run_command($cmd, noerr => 1, quiet => 0);
2949         die "QEMU flag querying VM exited with code " . $rc if $rc;
2950
2951         eval {
2952             my $cmd_result = mon_cmd(
2953                 $fakevmid,
2954                 'query-cpu-model-expansion',
2955                 type => 'full',
2956                 model => { name => 'host' }
2957             );
2958
2959             my $props = $cmd_result->{model}->{props};
2960             foreach my $prop (keys %$props) {
2961                 next if $props->{$prop} ne '1';
2962                 # QEMU returns some flags multiple times, with '_', '.' or '-'
2963                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
2964                 # We only keep those with underscores, to match /proc/cpuinfo
2965                 $prop =~ s/\.|-/_/g;
2966                 $flags->{$prop} = 1;
2967             }
2968         };
2969         my $err = $@;
2970
2971         # force stop with 10 sec timeout and 'nocheck'
2972         # always stop, even if QMP failed
2973         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
2974
2975         die $err if $err;
2976
2977         return [ sort keys %$flags ];
2978     };
2979
2980     # We need to query QEMU twice, since KVM and TCG have different supported flags
2981     PVE::QemuConfig->lock_config($fakevmid, sub {
2982         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
2983         warn "warning: failed querying supported tcg flags: $@\n" if $@;
2984
2985         if ($kvm_supported) {
2986             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
2987             warn "warning: failed querying supported kvm flags: $@\n" if $@;
2988         }
2989     });
2990
2991     return $flags;
2992 }
2993
2994 # Understood CPU flags are written to a file at 'pve-qemu' compile time
2995 my $understood_cpu_flag_dir = "/usr/share/kvm";
2996 sub query_understood_cpu_flags {
2997     my $arch = get_host_arch();
2998     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
2999
3000     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3001         if ! -e $filepath;
3002
3003     my $raw = file_get_contents($filepath);
3004     $raw =~ s/^\s+|\s+$//g;
3005     my @flags = split(/\s+/, $raw);
3006
3007     return \@flags;
3008 }
3009
3010 sub config_to_command {
3011     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu) = @_;
3012
3013     my $cmd = [];
3014     my $globalFlags = [];
3015     my $machineFlags = [];
3016     my $rtcFlags = [];
3017     my $devices = [];
3018     my $pciaddr = '';
3019     my $bridges = {};
3020     my $ostype = $conf->{ostype};
3021     my $winversion = windows_version($ostype);
3022     my $kvm = $conf->{kvm};
3023     my $nodename = nodename();
3024
3025     my $arch = get_vm_arch($conf);
3026     my $kvm_binary = get_command_for_arch($arch);
3027     my $kvmver = kvm_user_version($kvm_binary);
3028
3029     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3030         $kvmver //= "undefined";
3031         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3032     }
3033
3034     my $add_pve_version = min_version($kvmver, 4, 1);
3035
3036     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3037     my $machine_version = extract_version($machine_type, $kvmver);
3038     $kvm //= 1 if is_native($arch);
3039
3040     $machine_version =~ m/(\d+)\.(\d+)/;
3041     my ($machine_major, $machine_minor) = ($1, $2);
3042
3043     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3044         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3045     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3046         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3047             ." please upgrade node '$nodename'\n"
3048     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3049         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3050         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3051             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3052             ." node '$nodename'\n";
3053     }
3054
3055     # if a specific +pve version is required for a feature, use $version_guard
3056     # instead of min_version to allow machines to be run with the minimum
3057     # required version
3058     my $required_pve_version = 0;
3059     my $version_guard = sub {
3060         my ($major, $minor, $pve) = @_;
3061         return 0 if !min_version($machine_version, $major, $minor, $pve);
3062         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3063         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3064         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3065         return 1;
3066     };
3067
3068     if ($kvm && !defined kvm_version()) {
3069         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3070             ." or enable in BIOS.\n";
3071     }
3072
3073     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3074     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3075     my $use_old_bios_files = undef;
3076     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3077
3078     my $cpuunits = defined($conf->{cpuunits}) ?
3079             $conf->{cpuunits} : $defaults->{cpuunits};
3080
3081     push @$cmd, $kvm_binary;
3082
3083     push @$cmd, '-id', $vmid;
3084
3085     my $vmname = $conf->{name} || "vm$vmid";
3086
3087     push @$cmd, '-name', $vmname;
3088
3089     my $use_virtio = 0;
3090
3091     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3092     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3093     push @$cmd, '-mon', "chardev=qmp,mode=control";
3094
3095     if (min_version($machine_version, 2, 12)) {
3096         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3097         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3098     }
3099
3100     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3101
3102     push @$cmd, '-daemonize';
3103
3104     if ($conf->{smbios1}) {
3105         my $smbios_conf = parse_smbios1($conf->{smbios1});
3106         if ($smbios_conf->{base64}) {
3107             # Do not pass base64 flag to qemu
3108             delete $smbios_conf->{base64};
3109             my $smbios_string = "";
3110             foreach my $key (keys %$smbios_conf) {
3111                 my $value;
3112                 if ($key eq "uuid") {
3113                     $value = $smbios_conf->{uuid}
3114                 } else {
3115                     $value = decode_base64($smbios_conf->{$key});
3116                 }
3117                 # qemu accepts any binary data, only commas need escaping by double comma
3118                 $value =~ s/,/,,/g;
3119                 $smbios_string .= "," . $key . "=" . $value if $value;
3120             }
3121             push @$cmd, '-smbios', "type=1" . $smbios_string;
3122         } else {
3123             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3124         }
3125     }
3126
3127     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3128         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3129         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3130
3131         my ($path, $format);
3132         if (my $efidisk = $conf->{efidisk0}) {
3133             my $d = parse_drive('efidisk0', $efidisk);
3134             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3135             $format = $d->{format};
3136             if ($storeid) {
3137                 $path = PVE::Storage::path($storecfg, $d->{file});
3138                 if (!defined($format)) {
3139                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3140                     $format = qemu_img_format($scfg, $volname);
3141                 }
3142             } else {
3143                 $path = $d->{file};
3144                 die "efidisk format must be specified\n"
3145                     if !defined($format);
3146             }
3147         } else {
3148             warn "no efidisk configured! Using temporary efivars disk.\n";
3149             $path = "/tmp/$vmid-ovmf.fd";
3150             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3151             $format = 'raw';
3152         }
3153
3154         my $size_str = "";
3155
3156         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3157             $size_str = ",size=" . (-s $ovmf_vars);
3158         }
3159
3160         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3161         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0$size_str,file=$path";
3162     }
3163
3164     # load q35 config
3165     if ($q35) {
3166         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3167         if (min_version($machine_version, 4, 0)) {
3168             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3169         } else {
3170             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3171         }
3172     }
3173
3174     if ($conf->{vmgenid}) {
3175         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3176     }
3177
3178     # add usb controllers
3179     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3180         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3181     push @$devices, @usbcontrollers if @usbcontrollers;
3182     my $vga = parse_vga($conf->{vga});
3183
3184     my $qxlnum = vga_conf_has_spice($conf->{vga});
3185     $vga->{type} = 'qxl' if $qxlnum;
3186
3187     if (!$vga->{type}) {
3188         if ($arch eq 'aarch64') {
3189             $vga->{type} = 'virtio';
3190         } elsif (min_version($machine_version, 2, 9)) {
3191             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3192         } else {
3193             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3194         }
3195     }
3196
3197     # enable absolute mouse coordinates (needed by vnc)
3198     my $tablet;
3199     if (defined($conf->{tablet})) {
3200         $tablet = $conf->{tablet};
3201     } else {
3202         $tablet = $defaults->{tablet};
3203         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3204         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3205     }
3206
3207     if ($tablet) {
3208         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3209         my $kbd = print_keyboarddevice_full($conf, $arch);
3210         push @$devices, '-device', $kbd if defined($kbd);
3211     }
3212
3213     my $bootorder = {};
3214     my $boot = parse_property_string($boot_fmt, $conf->{boot}) if $conf->{boot};
3215     if (!defined($boot) || $boot->{legacy}) {
3216         $bootorder = bootorder_from_legacy($conf, $boot);
3217     } elsif ($boot->{order}) {
3218         # start at 100 to allow user to insert devices before us with -args
3219         my $i = 100;
3220         for my $dev (PVE::Tools::split_list($boot->{order})) {
3221             $bootorder->{$dev} = $i++;
3222         }
3223     }
3224
3225     # host pci device passthrough
3226     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3227         $vmid, $conf, $devices, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3228
3229     # usb devices
3230     my $usb_dev_features = {};
3231     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3232
3233     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3234         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3235     push @$devices, @usbdevices if @usbdevices;
3236
3237     # serial devices
3238     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3239         if (my $path = $conf->{"serial$i"}) {
3240             if ($path eq 'socket') {
3241                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3242                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3243                 # On aarch64, serial0 is the UART device. Qemu only allows
3244                 # connecting UART devices via the '-serial' command line, as
3245                 # the device has a fixed slot on the hardware...
3246                 if ($arch eq 'aarch64' && $i == 0) {
3247                     push @$devices, '-serial', "chardev:serial$i";
3248                 } else {
3249                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3250                 }
3251             } else {
3252                 die "no such serial device\n" if ! -c $path;
3253                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3254                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3255             }
3256         }
3257     }
3258
3259     # parallel devices
3260     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3261         if (my $path = $conf->{"parallel$i"}) {
3262             die "no such parallel device\n" if ! -c $path;
3263             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3264             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3265             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3266         }
3267     }
3268
3269     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3270         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3271         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3272         push @$devices, @$audio_devs;
3273     }
3274
3275     my $sockets = 1;
3276     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3277     $sockets = $conf->{sockets} if  $conf->{sockets};
3278
3279     my $cores = $conf->{cores} || 1;
3280
3281     my $maxcpus = $sockets * $cores;
3282
3283     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3284
3285     my $allowed_vcpus = $cpuinfo->{cpus};
3286
3287     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3288         if ($allowed_vcpus < $maxcpus);
3289
3290     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3291
3292         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3293         for (my $i = 2; $i <= $vcpus; $i++)  {
3294             my $cpustr = print_cpu_device($conf,$i);
3295             push @$cmd, '-device', $cpustr;
3296         }
3297
3298     } else {
3299
3300         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3301     }
3302     push @$cmd, '-nodefaults';
3303
3304     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3305
3306     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3307
3308     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3309
3310     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3311         push @$devices, '-device', print_vga_device(
3312             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3313         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3314         push @$cmd,  '-vnc', "unix:$socket,password";
3315     } else {
3316         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3317         push @$cmd, '-nographic';
3318     }
3319
3320     # time drift fix
3321     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3322     my $useLocaltime = $conf->{localtime};
3323
3324     if ($winversion >= 5) { # windows
3325         $useLocaltime = 1 if !defined($conf->{localtime});
3326
3327         # use time drift fix when acpi is enabled
3328         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3329             $tdf = 1 if !defined($conf->{tdf});
3330         }
3331     }
3332
3333     if ($winversion >= 6) {
3334         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3335         push @$cmd, '-no-hpet';
3336     }
3337
3338     push @$rtcFlags, 'driftfix=slew' if $tdf;
3339
3340     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3341         push @$rtcFlags, "base=$conf->{startdate}";
3342     } elsif ($useLocaltime) {
3343         push @$rtcFlags, 'base=localtime';
3344     }
3345
3346     if ($forcecpu) {
3347         push @$cmd, '-cpu', $forcecpu;
3348     } else {
3349         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3350     }
3351
3352     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3353
3354     push @$cmd, '-S' if $conf->{freeze};
3355
3356     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3357
3358     my $guest_agent = parse_guest_agent($conf);
3359
3360     if ($guest_agent->{enabled}) {
3361         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3362         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3363
3364         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3365             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3366             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3367             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3368         } elsif ($guest_agent->{type} eq 'isa') {
3369             push @$devices, '-device', "isa-serial,chardev=qga0";
3370         }
3371     }
3372
3373     my $rng = parse_rng($conf->{rng0}) if $conf->{rng0};
3374     if ($rng && &$version_guard(4, 1, 2)) {
3375         check_rng_source($rng->{source});
3376
3377         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3378         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3379         my $limiter_str = "";
3380         if ($max_bytes) {
3381             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3382         }
3383
3384         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3385         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3386         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3387     }
3388
3389     my $spice_port;
3390
3391     if ($qxlnum) {
3392         if ($qxlnum > 1) {
3393             if ($winversion){
3394                 for (my $i = 1; $i < $qxlnum; $i++){
3395                     push @$devices, '-device', print_vga_device(
3396                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3397                 }
3398             } else {
3399                 # assume other OS works like Linux
3400                 my ($ram, $vram) = ("134217728", "67108864");
3401                 if ($vga->{memory}) {
3402                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3403                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3404                 }
3405                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3406                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3407             }
3408         }
3409
3410         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3411
3412         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3413         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3414         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3415
3416         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3417         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3418         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3419
3420         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3421         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3422
3423         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3424         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3425         if ($spice_enhancement->{foldersharing}) {
3426             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3427             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3428         }
3429
3430         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3431         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3432             if $spice_enhancement->{videostreaming};
3433
3434         push @$devices, '-spice', "$spice_opts";
3435     }
3436
3437     # enable balloon by default, unless explicitly disabled
3438     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3439         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3440         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3441     }
3442
3443     if ($conf->{watchdog}) {
3444         my $wdopts = parse_watchdog($conf->{watchdog});
3445         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3446         my $watchdog = $wdopts->{model} || 'i6300esb';
3447         push @$devices, '-device', "$watchdog$pciaddr";
3448         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3449     }
3450
3451     my $vollist = [];
3452     my $scsicontroller = {};
3453     my $ahcicontroller = {};
3454     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3455
3456     # Add iscsi initiator name if available
3457     if (my $initiator = get_initiator_name()) {
3458         push @$devices, '-iscsi', "initiator-name=$initiator";
3459     }
3460
3461     PVE::QemuConfig->foreach_volume($conf, sub {
3462         my ($ds, $drive) = @_;
3463
3464         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3465             push @$vollist, $drive->{file};
3466         }
3467
3468         # ignore efidisk here, already added in bios/fw handling code above
3469         return if $drive->{interface} eq 'efidisk';
3470
3471         $use_virtio = 1 if $ds =~ m/^virtio/;
3472
3473         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3474
3475         if ($drive->{interface} eq 'virtio'){
3476            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3477         }
3478
3479         if ($drive->{interface} eq 'scsi') {
3480
3481             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3482
3483             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3484                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3485
3486             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3487             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3488
3489             my $iothread = '';
3490             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3491                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3492                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3493             } elsif ($drive->{iothread}) {
3494                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3495             }
3496
3497             my $queues = '';
3498             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3499                 $queues = ",num_queues=$drive->{queues}";
3500             }
3501
3502             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3503                 if !$scsicontroller->{$controller};
3504             $scsicontroller->{$controller}=1;
3505         }
3506
3507         if ($drive->{interface} eq 'sata') {
3508             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3509             $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3510             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3511                 if !$ahcicontroller->{$controller};
3512             $ahcicontroller->{$controller}=1;
3513         }
3514
3515         my $drive_cmd = print_drive_commandline_full($storecfg, $vmid, $drive);
3516         $drive_cmd .= ',readonly' if PVE::QemuConfig->is_template($conf);
3517
3518         push @$devices, '-drive',$drive_cmd;
3519         push @$devices, '-device', print_drivedevice_full(
3520             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3521     });
3522
3523     for (my $i = 0; $i < $MAX_NETS; $i++) {
3524         my $netname = "net$i";
3525
3526         next if !$conf->{$netname};
3527         my $d = parse_net($conf->{$netname});
3528         next if !$d;
3529
3530         $use_virtio = 1 if $d->{model} eq 'virtio';
3531
3532         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
3533
3534         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
3535         push @$devices, '-netdev', $netdevfull;
3536
3537         my $netdevicefull = print_netdevice_full(
3538             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
3539
3540         push @$devices, '-device', $netdevicefull;
3541     }
3542
3543     if ($conf->{ivshmem}) {
3544         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3545
3546         my $bus;
3547         if ($q35) {
3548             $bus = print_pcie_addr("ivshmem");
3549         } else {
3550             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3551         }
3552
3553         my $ivshmem_name = $ivshmem->{name} // $vmid;
3554         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3555
3556         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3557         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
3558             .",size=$ivshmem->{size}M";
3559     }
3560
3561     # pci.4 is nested in pci.1
3562     $bridges->{1} = 1 if $bridges->{4};
3563
3564     if (!$q35) {
3565         # add pci bridges
3566         if (min_version($machine_version, 2, 3)) {
3567            $bridges->{1} = 1;
3568            $bridges->{2} = 1;
3569         }
3570
3571         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3572
3573     }
3574
3575     for my $k (sort {$b cmp $a} keys %$bridges) {
3576         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3577
3578         my $k_name = $k;
3579         if ($k == 2 && $legacy_igd) {
3580             $k_name = "$k-igd";
3581         }
3582         $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
3583
3584         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3585         if ($q35) {
3586             # add after -readconfig pve-q35.cfg
3587             splice @$devices, 2, 0, '-device', $devstr;
3588         } else {
3589             unshift @$devices, '-device', $devstr if $k > 0;
3590         }
3591     }
3592
3593     if (!$kvm) {
3594         push @$machineFlags, 'accel=tcg';
3595     }
3596
3597     my $machine_type_min = $machine_type;
3598     if ($add_pve_version) {
3599         $machine_type_min =~ s/\+pve\d+$//;
3600         $machine_type_min .= "+pve$required_pve_version";
3601     }
3602     push @$machineFlags, "type=${machine_type_min}";
3603
3604     push @$cmd, @$devices;
3605     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
3606     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
3607     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
3608
3609     if (my $vmstate = $conf->{vmstate}) {
3610         my $statepath = PVE::Storage::path($storecfg, $vmstate);
3611         push @$vollist, $vmstate;
3612         push @$cmd, '-loadstate', $statepath;
3613         print "activating and using '$vmstate' as vmstate\n";
3614     }
3615
3616     # add custom args
3617     if ($conf->{args}) {
3618         my $aa = PVE::Tools::split_args($conf->{args});
3619         push @$cmd, @$aa;
3620     }
3621
3622     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3623 }
3624
3625 sub check_rng_source {
3626     my ($source) = @_;
3627
3628     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3629     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
3630         if ! -e $source;
3631
3632     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
3633     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
3634         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
3635         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
3636         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
3637             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
3638             ." to the host.\n";
3639     }
3640 }
3641
3642 sub spice_port {
3643     my ($vmid) = @_;
3644
3645     my $res = mon_cmd($vmid, 'query-spice');
3646
3647     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3648 }
3649
3650 sub vm_devices_list {
3651     my ($vmid) = @_;
3652
3653     my $res = mon_cmd($vmid, 'query-pci');
3654     my $devices_to_check = [];
3655     my $devices = {};
3656     foreach my $pcibus (@$res) {
3657         push @$devices_to_check, @{$pcibus->{devices}},
3658     }
3659
3660     while (@$devices_to_check) {
3661         my $to_check = [];
3662         for my $d (@$devices_to_check) {
3663             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3664             next if !$d->{'pci_bridge'};
3665
3666             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3667             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3668         }
3669         $devices_to_check = $to_check;
3670     }
3671
3672     my $resblock = mon_cmd($vmid, 'query-block');
3673     foreach my $block (@$resblock) {
3674         if($block->{device} =~ m/^drive-(\S+)/){
3675                 $devices->{$1} = 1;
3676         }
3677     }
3678
3679     my $resmice = mon_cmd($vmid, 'query-mice');
3680     foreach my $mice (@$resmice) {
3681         if ($mice->{name} eq 'QEMU HID Tablet') {
3682             $devices->{tablet} = 1;
3683             last;
3684         }
3685     }
3686
3687     # for usb devices there is no query-usb
3688     # but we can iterate over the entries in
3689     # qom-list path=/machine/peripheral
3690     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3691     foreach my $per (@$resperipheral) {
3692         if ($per->{name} =~ m/^usb\d+$/) {
3693             $devices->{$per->{name}} = 1;
3694         }
3695     }
3696
3697     return $devices;
3698 }
3699
3700 sub vm_deviceplug {
3701     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3702
3703     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3704
3705     my $devices_list = vm_devices_list($vmid);
3706     return 1 if defined($devices_list->{$deviceid});
3707
3708     # add PCI bridge if we need it for the device
3709     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
3710
3711     if ($deviceid eq 'tablet') {
3712
3713         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
3714
3715     } elsif ($deviceid eq 'keyboard') {
3716
3717         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
3718
3719     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3720
3721         die "usb hotplug currently not reliable\n";
3722         # since we can't reliably hot unplug all added usb devices
3723         # and usb passthrough disables live migration
3724         # we disable usb hotplugging for now
3725         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3726
3727     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3728
3729         qemu_iothread_add($vmid, $deviceid, $device);
3730
3731         qemu_driveadd($storecfg, $vmid, $device);
3732         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
3733
3734         qemu_deviceadd($vmid, $devicefull);
3735         eval { qemu_deviceaddverify($vmid, $deviceid); };
3736         if (my $err = $@) {
3737             eval { qemu_drivedel($vmid, $deviceid); };
3738             warn $@ if $@;
3739             die $err;
3740         }
3741
3742     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3743
3744
3745         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3746         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3747         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3748
3749         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3750
3751         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3752             qemu_iothread_add($vmid, $deviceid, $device);
3753             $devicefull .= ",iothread=iothread-$deviceid";
3754         }
3755
3756         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3757             $devicefull .= ",num_queues=$device->{queues}";
3758         }
3759
3760         qemu_deviceadd($vmid, $devicefull);
3761         qemu_deviceaddverify($vmid, $deviceid);
3762
3763     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3764
3765         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
3766         qemu_driveadd($storecfg, $vmid, $device);
3767
3768         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
3769         eval { qemu_deviceadd($vmid, $devicefull); };
3770         if (my $err = $@) {
3771             eval { qemu_drivedel($vmid, $deviceid); };
3772             warn $@ if $@;
3773             die $err;
3774         }
3775
3776     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3777
3778         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
3779
3780         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
3781         my $use_old_bios_files = undef;
3782         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3783
3784         my $netdevicefull = print_netdevice_full(
3785             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
3786         qemu_deviceadd($vmid, $netdevicefull);
3787         eval {
3788             qemu_deviceaddverify($vmid, $deviceid);
3789             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
3790         };
3791         if (my $err = $@) {
3792             eval { qemu_netdevdel($vmid, $deviceid); };
3793             warn $@ if $@;
3794             die $err;
3795         }
3796
3797     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3798
3799         my $bridgeid = $2;
3800         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3801         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3802
3803         qemu_deviceadd($vmid, $devicefull);
3804         qemu_deviceaddverify($vmid, $deviceid);
3805
3806     } else {
3807         die "can't hotplug device '$deviceid'\n";
3808     }
3809
3810     return 1;
3811 }
3812
3813 # fixme: this should raise exceptions on error!
3814 sub vm_deviceunplug {
3815     my ($vmid, $conf, $deviceid) = @_;
3816
3817     my $devices_list = vm_devices_list($vmid);
3818     return 1 if !defined($devices_list->{$deviceid});
3819
3820     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
3821     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
3822
3823     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
3824
3825         qemu_devicedel($vmid, $deviceid);
3826
3827     } elsif ($deviceid =~ m/^usb\d+$/) {
3828
3829         die "usb hotplug currently not reliable\n";
3830         # when unplugging usb devices this way,
3831         # there may be remaining usb controllers/hubs
3832         # so we disable it for now
3833         qemu_devicedel($vmid, $deviceid);
3834         qemu_devicedelverify($vmid, $deviceid);
3835
3836     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3837
3838         qemu_devicedel($vmid, $deviceid);
3839         qemu_devicedelverify($vmid, $deviceid);
3840         qemu_drivedel($vmid, $deviceid);
3841         qemu_iothread_del($conf, $vmid, $deviceid);
3842
3843     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3844
3845         qemu_devicedel($vmid, $deviceid);
3846         qemu_devicedelverify($vmid, $deviceid);
3847         qemu_iothread_del($conf, $vmid, $deviceid);
3848
3849     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3850
3851         qemu_devicedel($vmid, $deviceid);
3852         qemu_drivedel($vmid, $deviceid);
3853         qemu_deletescsihw($conf, $vmid, $deviceid);
3854
3855     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3856
3857         qemu_devicedel($vmid, $deviceid);
3858         qemu_devicedelverify($vmid, $deviceid);
3859         qemu_netdevdel($vmid, $deviceid);
3860
3861     } else {
3862         die "can't unplug device '$deviceid'\n";
3863     }
3864
3865     return 1;
3866 }
3867
3868 sub qemu_deviceadd {
3869     my ($vmid, $devicefull) = @_;
3870
3871     $devicefull = "driver=".$devicefull;
3872     my %options =  split(/[=,]/, $devicefull);
3873
3874     mon_cmd($vmid, "device_add" , %options);
3875 }
3876
3877 sub qemu_devicedel {
3878     my ($vmid, $deviceid) = @_;
3879
3880     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
3881 }
3882
3883 sub qemu_iothread_add {
3884     my($vmid, $deviceid, $device) = @_;
3885
3886     if ($device->{iothread}) {
3887         my $iothreads = vm_iothreads_list($vmid);
3888         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3889     }
3890 }
3891
3892 sub qemu_iothread_del {
3893     my($conf, $vmid, $deviceid) = @_;
3894
3895     my $confid = $deviceid;
3896     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
3897         $confid = 'scsi' . $1;
3898     }
3899     my $device = parse_drive($confid, $conf->{$confid});
3900     if ($device->{iothread}) {
3901         my $iothreads = vm_iothreads_list($vmid);
3902         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3903     }
3904 }
3905
3906 sub qemu_objectadd {
3907     my($vmid, $objectid, $qomtype) = @_;
3908
3909     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3910
3911     return 1;
3912 }
3913
3914 sub qemu_objectdel {
3915     my($vmid, $objectid) = @_;
3916
3917     mon_cmd($vmid, "object-del", id => $objectid);
3918
3919     return 1;
3920 }
3921
3922 sub qemu_driveadd {
3923     my ($storecfg, $vmid, $device) = @_;
3924
3925     my $drive = print_drive_commandline_full($storecfg, $vmid, $device);
3926     $drive =~ s/\\/\\\\/g;
3927     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
3928
3929     # If the command succeeds qemu prints: "OK"
3930     return 1 if $ret =~ m/OK/s;
3931
3932     die "adding drive failed: $ret\n";
3933 }
3934
3935 sub qemu_drivedel {
3936     my($vmid, $deviceid) = @_;
3937
3938     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
3939     $ret =~ s/^\s+//;
3940
3941     return 1 if $ret eq "";
3942
3943     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3944     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3945
3946     die "deleting drive $deviceid failed : $ret\n";
3947 }
3948
3949 sub qemu_deviceaddverify {
3950     my ($vmid, $deviceid) = @_;
3951
3952     for (my $i = 0; $i <= 5; $i++) {
3953          my $devices_list = vm_devices_list($vmid);
3954          return 1 if defined($devices_list->{$deviceid});
3955          sleep 1;
3956     }
3957
3958     die "error on hotplug device '$deviceid'\n";
3959 }
3960
3961
3962 sub qemu_devicedelverify {
3963     my ($vmid, $deviceid) = @_;
3964
3965     # need to verify that the device is correctly removed as device_del
3966     # is async and empty return is not reliable
3967
3968     for (my $i = 0; $i <= 5; $i++) {
3969          my $devices_list = vm_devices_list($vmid);
3970          return 1 if !defined($devices_list->{$deviceid});
3971          sleep 1;
3972     }
3973
3974     die "error on hot-unplugging device '$deviceid'\n";
3975 }
3976
3977 sub qemu_findorcreatescsihw {
3978     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
3979
3980     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3981
3982     my $scsihwid="$controller_prefix$controller";
3983     my $devices_list = vm_devices_list($vmid);
3984
3985     if(!defined($devices_list->{$scsihwid})) {
3986         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
3987     }
3988
3989     return 1;
3990 }
3991
3992 sub qemu_deletescsihw {
3993     my ($conf, $vmid, $opt) = @_;
3994
3995     my $device = parse_drive($opt, $conf->{$opt});
3996
3997     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3998         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3999         return 1;
4000     }
4001
4002     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4003
4004     my $devices_list = vm_devices_list($vmid);
4005     foreach my $opt (keys %{$devices_list}) {
4006         if (is_valid_drivename($opt)) {
4007             my $drive = parse_drive($opt, $conf->{$opt});
4008             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4009                 return 1;
4010             }
4011         }
4012     }
4013
4014     my $scsihwid="scsihw$controller";
4015
4016     vm_deviceunplug($vmid, $conf, $scsihwid);
4017
4018     return 1;
4019 }
4020
4021 sub qemu_add_pci_bridge {
4022     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4023
4024     my $bridges = {};
4025
4026     my $bridgeid;
4027
4028     print_pci_addr($device, $bridges, $arch, $machine_type);
4029
4030     while (my ($k, $v) = each %$bridges) {
4031         $bridgeid = $k;
4032     }
4033     return 1 if !defined($bridgeid) || $bridgeid < 1;
4034
4035     my $bridge = "pci.$bridgeid";
4036     my $devices_list = vm_devices_list($vmid);
4037
4038     if (!defined($devices_list->{$bridge})) {
4039         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4040     }
4041
4042     return 1;
4043 }
4044
4045 sub qemu_set_link_status {
4046     my ($vmid, $device, $up) = @_;
4047
4048     mon_cmd($vmid, "set_link", name => $device,
4049                up => $up ? JSON::true : JSON::false);
4050 }
4051
4052 sub qemu_netdevadd {
4053     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4054
4055     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4056     my %options =  split(/[=,]/, $netdev);
4057
4058     if (defined(my $vhost = $options{vhost})) {
4059         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4060     }
4061
4062     if (defined(my $queues = $options{queues})) {
4063         $options{queues} = $queues + 0;
4064     }
4065
4066     mon_cmd($vmid, "netdev_add",  %options);
4067     return 1;
4068 }
4069
4070 sub qemu_netdevdel {
4071     my ($vmid, $deviceid) = @_;
4072
4073     mon_cmd($vmid, "netdev_del", id => $deviceid);
4074 }
4075
4076 sub qemu_usb_hotplug {
4077     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4078
4079     return if !$device;
4080
4081     # remove the old one first
4082     vm_deviceunplug($vmid, $conf, $deviceid);
4083
4084     # check if xhci controller is necessary and available
4085     if ($device->{usb3}) {
4086
4087         my $devicelist = vm_devices_list($vmid);
4088
4089         if (!$devicelist->{xhci}) {
4090             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4091             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4092         }
4093     }
4094     my $d = parse_usb_device($device->{host});
4095     $d->{usb3} = $device->{usb3};
4096
4097     # add the new one
4098     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4099 }
4100
4101 sub qemu_cpu_hotplug {
4102     my ($vmid, $conf, $vcpus) = @_;
4103
4104     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4105
4106     my $sockets = 1;
4107     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4108     $sockets = $conf->{sockets} if  $conf->{sockets};
4109     my $cores = $conf->{cores} || 1;
4110     my $maxcpus = $sockets * $cores;
4111
4112     $vcpus = $maxcpus if !$vcpus;
4113
4114     die "you can't add more vcpus than maxcpus\n"
4115         if $vcpus > $maxcpus;
4116
4117     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4118
4119     if ($vcpus < $currentvcpus) {
4120
4121         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4122
4123             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4124                 qemu_devicedel($vmid, "cpu$i");
4125                 my $retry = 0;
4126                 my $currentrunningvcpus = undef;
4127                 while (1) {
4128                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4129                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4130                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4131                     $retry++;
4132                     sleep 1;
4133                 }
4134                 #update conf after each succesfull cpu unplug
4135                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4136                 PVE::QemuConfig->write_config($vmid, $conf);
4137             }
4138         } else {
4139             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4140         }
4141
4142         return;
4143     }
4144
4145     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4146     die "vcpus in running vm does not match its configuration\n"
4147         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4148
4149     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4150
4151         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4152             my $cpustr = print_cpu_device($conf, $i);
4153             qemu_deviceadd($vmid, $cpustr);
4154
4155             my $retry = 0;
4156             my $currentrunningvcpus = undef;
4157             while (1) {
4158                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4159                 last if scalar(@{$currentrunningvcpus}) == $i;
4160                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4161                 sleep 1;
4162                 $retry++;
4163             }
4164             #update conf after each succesfull cpu hotplug
4165             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4166             PVE::QemuConfig->write_config($vmid, $conf);
4167         }
4168     } else {
4169
4170         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4171             mon_cmd($vmid, "cpu-add", id => int($i));
4172         }
4173     }
4174 }
4175
4176 sub qemu_block_set_io_throttle {
4177     my ($vmid, $deviceid,
4178         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4179         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4180         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4181         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4182
4183     return if !check_running($vmid) ;
4184
4185     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4186         bps => int($bps),
4187         bps_rd => int($bps_rd),
4188         bps_wr => int($bps_wr),
4189         iops => int($iops),
4190         iops_rd => int($iops_rd),
4191         iops_wr => int($iops_wr),
4192         bps_max => int($bps_max),
4193         bps_rd_max => int($bps_rd_max),
4194         bps_wr_max => int($bps_wr_max),
4195         iops_max => int($iops_max),
4196         iops_rd_max => int($iops_rd_max),
4197         iops_wr_max => int($iops_wr_max),
4198         bps_max_length => int($bps_max_length),
4199         bps_rd_max_length => int($bps_rd_max_length),
4200         bps_wr_max_length => int($bps_wr_max_length),
4201         iops_max_length => int($iops_max_length),
4202         iops_rd_max_length => int($iops_rd_max_length),
4203         iops_wr_max_length => int($iops_wr_max_length),
4204     );
4205
4206 }
4207
4208 # old code, only used to shutdown old VM after update
4209 sub __read_avail {
4210     my ($fh, $timeout) = @_;
4211
4212     my $sel = new IO::Select;
4213     $sel->add($fh);
4214
4215     my $res = '';
4216     my $buf;
4217
4218     my @ready;
4219     while (scalar (@ready = $sel->can_read($timeout))) {
4220         my $count;
4221         if ($count = $fh->sysread($buf, 8192)) {
4222             if ($buf =~ /^(.*)\(qemu\) $/s) {
4223                 $res .= $1;
4224                 last;
4225             } else {
4226                 $res .= $buf;
4227             }
4228         } else {
4229             if (!defined($count)) {
4230                 die "$!\n";
4231             }
4232             last;
4233         }
4234     }
4235
4236     die "monitor read timeout\n" if !scalar(@ready);
4237
4238     return $res;
4239 }
4240
4241 sub qemu_block_resize {
4242     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4243
4244     my $running = check_running($vmid);
4245
4246     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4247
4248     return if !$running;
4249
4250     my $padding = (1024 - $size % 1024) % 1024;
4251     $size = $size + $padding;
4252
4253     mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4254
4255 }
4256
4257 sub qemu_volume_snapshot {
4258     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4259
4260     my $running = check_running($vmid);
4261
4262     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4263         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4264     } else {
4265         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4266     }
4267 }
4268
4269 sub qemu_volume_snapshot_delete {
4270     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4271
4272     my $running = check_running($vmid);
4273
4274     if($running) {
4275
4276         $running = undef;
4277         my $conf = PVE::QemuConfig->load_config($vmid);
4278         PVE::QemuConfig->foreach_volume($conf, sub {
4279             my ($ds, $drive) = @_;
4280             $running = 1 if $drive->{file} eq $volid;
4281         });
4282     }
4283
4284     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4285         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4286     } else {
4287         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4288     }
4289 }
4290
4291 sub set_migration_caps {
4292     my ($vmid) = @_;
4293
4294     my $cap_ref = [];
4295
4296     my $enabled_cap = {
4297         "auto-converge" => 1,
4298         "xbzrle" => 1,
4299         "x-rdma-pin-all" => 0,
4300         "zero-blocks" => 0,
4301         "compress" => 0
4302     };
4303
4304     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4305
4306     for my $supported_capability (@$supported_capabilities) {
4307         push @$cap_ref, {
4308             capability => $supported_capability->{capability},
4309             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4310         };
4311     }
4312
4313     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4314 }
4315
4316 sub foreach_volid {
4317     my ($conf, $func, @param) = @_;
4318
4319     my $volhash = {};
4320
4321     my $test_volid = sub {
4322         my ($key, $drive, $snapname) = @_;
4323
4324         my $volid = $drive->{file};
4325         return if !$volid;
4326
4327         $volhash->{$volid}->{cdrom} //= 1;
4328         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4329
4330         my $replicate = $drive->{replicate} // 1;
4331         $volhash->{$volid}->{replicate} //= 0;
4332         $volhash->{$volid}->{replicate} = 1 if $replicate;
4333
4334         $volhash->{$volid}->{shared} //= 0;
4335         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4336
4337         $volhash->{$volid}->{referenced_in_config} //= 0;
4338         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4339
4340         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4341             if defined($snapname);
4342
4343         my $size = $drive->{size};
4344         $volhash->{$volid}->{size} //= $size if $size;
4345
4346         $volhash->{$volid}->{is_vmstate} //= 0;
4347         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4348
4349         $volhash->{$volid}->{is_unused} //= 0;
4350         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4351     };
4352
4353     my $include_opts = {
4354         extra_keys => ['vmstate'],
4355         include_unused => 1,
4356     };
4357
4358     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4359     foreach my $snapname (keys %{$conf->{snapshots}}) {
4360         my $snap = $conf->{snapshots}->{$snapname};
4361         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4362     }
4363
4364     foreach my $volid (keys %$volhash) {
4365         &$func($volid, $volhash->{$volid}, @param);
4366     }
4367 }
4368
4369 my $fast_plug_option = {
4370     'lock' => 1,
4371     'name' => 1,
4372     'onboot' => 1,
4373     'shares' => 1,
4374     'startup' => 1,
4375     'description' => 1,
4376     'protection' => 1,
4377     'vmstatestorage' => 1,
4378     'hookscript' => 1,
4379     'tags' => 1,
4380 };
4381
4382 # hotplug changes in [PENDING]
4383 # $selection hash can be used to only apply specified options, for
4384 # example: { cores => 1 } (only apply changed 'cores')
4385 # $errors ref is used to return error messages
4386 sub vmconfig_hotplug_pending {
4387     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4388
4389     my $defaults = load_defaults();
4390     my $arch = get_vm_arch($conf);
4391     my $machine_type = get_vm_machine($conf, undef, $arch);
4392
4393     # commit values which do not have any impact on running VM first
4394     # Note: those option cannot raise errors, we we do not care about
4395     # $selection and always apply them.
4396
4397     my $add_error = sub {
4398         my ($opt, $msg) = @_;
4399         $errors->{$opt} = "hotplug problem - $msg";
4400     };
4401
4402     my $changes = 0;
4403     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4404         if ($fast_plug_option->{$opt}) {
4405             $conf->{$opt} = $conf->{pending}->{$opt};
4406             delete $conf->{pending}->{$opt};
4407             $changes = 1;
4408         }
4409     }
4410
4411     if ($changes) {
4412         PVE::QemuConfig->write_config($vmid, $conf);
4413     }
4414
4415     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4416
4417     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4418     foreach my $opt (sort keys %$pending_delete_hash) {
4419         next if $selection && !$selection->{$opt};
4420         my $force = $pending_delete_hash->{$opt}->{force};
4421         eval {
4422             if ($opt eq 'hotplug') {
4423                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4424             } elsif ($opt eq 'tablet') {
4425                 die "skip\n" if !$hotplug_features->{usb};
4426                 if ($defaults->{tablet}) {
4427                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4428                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4429                         if $arch eq 'aarch64';
4430                 } else {
4431                     vm_deviceunplug($vmid, $conf, 'tablet');
4432                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4433                 }
4434             } elsif ($opt =~ m/^usb\d+/) {
4435                 die "skip\n";
4436                 # since we cannot reliably hot unplug usb devices
4437                 # we are disabling it
4438                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4439                 vm_deviceunplug($vmid, $conf, $opt);
4440             } elsif ($opt eq 'vcpus') {
4441                 die "skip\n" if !$hotplug_features->{cpu};
4442                 qemu_cpu_hotplug($vmid, $conf, undef);
4443             } elsif ($opt eq 'balloon') {
4444                 # enable balloon device is not hotpluggable
4445                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4446                 # here we reset the ballooning value to memory
4447                 my $balloon = $conf->{memory} || $defaults->{memory};
4448                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4449             } elsif ($fast_plug_option->{$opt}) {
4450                 # do nothing
4451             } elsif ($opt =~ m/^net(\d+)$/) {
4452                 die "skip\n" if !$hotplug_features->{network};
4453                 vm_deviceunplug($vmid, $conf, $opt);
4454             } elsif (is_valid_drivename($opt)) {
4455                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4456                 vm_deviceunplug($vmid, $conf, $opt);
4457                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4458             } elsif ($opt =~ m/^memory$/) {
4459                 die "skip\n" if !$hotplug_features->{memory};
4460                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4461             } elsif ($opt eq 'cpuunits') {
4462                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4463             } elsif ($opt eq 'cpulimit') {
4464                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4465             } else {
4466                 die "skip\n";
4467             }
4468         };
4469         if (my $err = $@) {
4470             &$add_error($opt, $err) if $err ne "skip\n";
4471         } else {
4472             delete $conf->{$opt};
4473             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4474         }
4475     }
4476
4477     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4478     $apply_pending_cloudinit = sub {
4479         return if $apply_pending_cloudinit_done; # once is enough
4480         $apply_pending_cloudinit_done = 1; # once is enough
4481
4482         my ($key, $value) = @_;
4483
4484         my @cloudinit_opts = keys %$confdesc_cloudinit;
4485         foreach my $opt (keys %{$conf->{pending}}) {
4486             next if !grep { $_ eq $opt } @cloudinit_opts;
4487             $conf->{$opt} = delete $conf->{pending}->{$opt};
4488         }
4489
4490         my $new_conf = { %$conf };
4491         $new_conf->{$key} = $value;
4492         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4493     };
4494
4495     foreach my $opt (keys %{$conf->{pending}}) {
4496         next if $selection && !$selection->{$opt};
4497         my $value = $conf->{pending}->{$opt};
4498         eval {
4499             if ($opt eq 'hotplug') {
4500                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4501             } elsif ($opt eq 'tablet') {
4502                 die "skip\n" if !$hotplug_features->{usb};
4503                 if ($value == 1) {
4504                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4505                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4506                         if $arch eq 'aarch64';
4507                 } elsif ($value == 0) {
4508                     vm_deviceunplug($vmid, $conf, 'tablet');
4509                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4510                 }
4511             } elsif ($opt =~ m/^usb\d+$/) {
4512                 die "skip\n";
4513                 # since we cannot reliably hot unplug usb devices
4514                 # we are disabling it
4515                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4516                 my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4517                 die "skip\n" if !$d;
4518                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4519             } elsif ($opt eq 'vcpus') {
4520                 die "skip\n" if !$hotplug_features->{cpu};
4521                 qemu_cpu_hotplug($vmid, $conf, $value);
4522             } elsif ($opt eq 'balloon') {
4523                 # enable/disable balloning device is not hotpluggable
4524                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4525                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4526                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4527
4528                 # allow manual ballooning if shares is set to zero
4529                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4530                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4531                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4532                 }
4533             } elsif ($opt =~ m/^net(\d+)$/) {
4534                 # some changes can be done without hotplug
4535                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4536                                     $vmid, $opt, $value, $arch, $machine_type);
4537             } elsif (is_valid_drivename($opt)) {
4538                 die "skip\n" if $opt eq 'efidisk0';
4539                 # some changes can be done without hotplug
4540                 my $drive = parse_drive($opt, $value);
4541                 if (drive_is_cloudinit($drive)) {
4542                     &$apply_pending_cloudinit($opt, $value);
4543                 }
4544                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4545                                      $vmid, $opt, $value, $arch, $machine_type);
4546             } elsif ($opt =~ m/^memory$/) { #dimms
4547                 die "skip\n" if !$hotplug_features->{memory};
4548                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4549             } elsif ($opt eq 'cpuunits') {
4550                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4551             } elsif ($opt eq 'cpulimit') {
4552                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4553                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4554             } else {
4555                 die "skip\n";  # skip non-hot-pluggable options
4556             }
4557         };
4558         if (my $err = $@) {
4559             &$add_error($opt, $err) if $err ne "skip\n";
4560         } else {
4561             $conf->{$opt} = $value;
4562             delete $conf->{pending}->{$opt};
4563         }
4564     }
4565
4566     PVE::QemuConfig->write_config($vmid, $conf);
4567 }
4568
4569 sub try_deallocate_drive {
4570     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4571
4572     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4573         my $volid = $drive->{file};
4574         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4575             my $sid = PVE::Storage::parse_volume_id($volid);
4576             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4577
4578             # check if the disk is really unused
4579             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4580                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4581             PVE::Storage::vdisk_free($storecfg, $volid);
4582             return 1;
4583         } else {
4584             # If vm is not owner of this disk remove from config
4585             return 1;
4586         }
4587     }
4588
4589     return;
4590 }
4591
4592 sub vmconfig_delete_or_detach_drive {
4593     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4594
4595     my $drive = parse_drive($opt, $conf->{$opt});
4596
4597     my $rpcenv = PVE::RPCEnvironment::get();
4598     my $authuser = $rpcenv->get_user();
4599
4600     if ($force) {
4601         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4602         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4603     } else {
4604         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4605     }
4606 }
4607
4608
4609
4610 sub vmconfig_apply_pending {
4611     my ($vmid, $conf, $storecfg, $errors) = @_;
4612
4613     my $add_apply_error = sub {
4614         my ($opt, $msg) = @_;
4615         my $err_msg = "unable to apply pending change $opt : $msg";
4616         $errors->{$opt} = $err_msg;
4617         warn $err_msg;
4618     };
4619
4620     # cold plug
4621
4622     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4623     foreach my $opt (sort keys %$pending_delete_hash) {
4624         my $force = $pending_delete_hash->{$opt}->{force};
4625         eval {
4626             if ($opt =~ m/^unused/) {
4627                 die "internal error";
4628             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4629                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4630             }
4631         };
4632         if (my $err = $@) {
4633             $add_apply_error->($opt, $err);
4634         } else {
4635             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4636             delete $conf->{$opt};
4637         }
4638     }
4639
4640     PVE::QemuConfig->cleanup_pending($conf);
4641
4642     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4643         next if $opt eq 'delete'; # just to be sure
4644         eval {
4645             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4646                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4647             }
4648         };
4649         if (my $err = $@) {
4650             $add_apply_error->($opt, $err);
4651         } else {
4652             $conf->{$opt} = delete $conf->{pending}->{$opt};
4653         }
4654     }
4655
4656     # write all changes at once to avoid unnecessary i/o
4657     PVE::QemuConfig->write_config($vmid, $conf);
4658 }
4659
4660 sub vmconfig_update_net {
4661     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4662
4663     my $newnet = parse_net($value);
4664
4665     if ($conf->{$opt}) {
4666         my $oldnet = parse_net($conf->{$opt});
4667
4668         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4669             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4670             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4671             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4672
4673             # for non online change, we try to hot-unplug
4674             die "skip\n" if !$hotplug;
4675             vm_deviceunplug($vmid, $conf, $opt);
4676         } else {
4677
4678             die "internal error" if $opt !~ m/net(\d+)/;
4679             my $iface = "tap${vmid}i$1";
4680
4681             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4682                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4683                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4684                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4685                 PVE::Network::tap_unplug($iface);
4686
4687                 if ($have_sdn) {
4688                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4689                 } else {
4690                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4691                 }
4692             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4693                 # Rate can be applied on its own but any change above needs to
4694                 # include the rate in tap_plug since OVS resets everything.
4695                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4696             }
4697
4698             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4699                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4700             }
4701
4702             return 1;
4703         }
4704     }
4705
4706     if ($hotplug) {
4707         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4708     } else {
4709         die "skip\n";
4710     }
4711 }
4712
4713 sub vmconfig_update_disk {
4714     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4715
4716     my $drive = parse_drive($opt, $value);
4717
4718     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
4719         my $media = $drive->{media} || 'disk';
4720         my $oldmedia = $old_drive->{media} || 'disk';
4721         die "unable to change media type\n" if $media ne $oldmedia;
4722
4723         if (!drive_is_cdrom($old_drive)) {
4724
4725             if ($drive->{file} ne $old_drive->{file}) {
4726
4727                 die "skip\n" if !$hotplug;
4728
4729                 # unplug and register as unused
4730                 vm_deviceunplug($vmid, $conf, $opt);
4731                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4732
4733             } else {
4734                 # update existing disk
4735
4736                 # skip non hotpluggable value
4737                 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4738                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4739                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4740                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
4741                     safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
4742                     die "skip\n";
4743                 }
4744
4745                 # apply throttle
4746                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4747                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4748                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4749                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4750                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4751                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4752                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4753                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4754                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4755                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4756                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4757                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4758                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4759                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4760                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4761                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4762                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4763                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4764
4765                     qemu_block_set_io_throttle(
4766                         $vmid,"drive-$opt",
4767                         ($drive->{mbps} || 0)*1024*1024,
4768                         ($drive->{mbps_rd} || 0)*1024*1024,
4769                         ($drive->{mbps_wr} || 0)*1024*1024,
4770                          $drive->{iops} || 0,
4771                          $drive->{iops_rd} || 0,
4772                          $drive->{iops_wr} || 0,
4773                         ($drive->{mbps_max} || 0)*1024*1024,
4774                         ($drive->{mbps_rd_max} || 0)*1024*1024,
4775                         ($drive->{mbps_wr_max} || 0)*1024*1024,
4776                          $drive->{iops_max} || 0,
4777                          $drive->{iops_rd_max} || 0,
4778                          $drive->{iops_wr_max} || 0,
4779                          $drive->{bps_max_length} || 1,
4780                          $drive->{bps_rd_max_length} || 1,
4781                          $drive->{bps_wr_max_length} || 1,
4782                          $drive->{iops_max_length} || 1,
4783                          $drive->{iops_rd_max_length} || 1,
4784                          $drive->{iops_wr_max_length} || 1,
4785                     );
4786
4787                 }
4788
4789                 return 1;
4790             }
4791
4792         } else { # cdrom
4793
4794             if ($drive->{file} eq 'none') {
4795                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4796                 if (drive_is_cloudinit($old_drive)) {
4797                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4798                 }
4799             } else {
4800                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4801
4802                 # force eject if locked
4803                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4804
4805                 if ($path) {
4806                     mon_cmd($vmid, "blockdev-change-medium",
4807                         id => "$opt", filename => "$path");
4808                 }
4809             }
4810
4811             return 1;
4812         }
4813     }
4814
4815     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4816     # hotplug new disks
4817     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4818     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
4819 }
4820
4821 # called in locked context by incoming migration
4822 sub vm_migrate_get_nbd_disks {
4823     my ($storecfg, $conf, $replicated_volumes) = @_;
4824
4825     my $local_volumes = {};
4826     PVE::QemuConfig->foreach_volume($conf, sub {
4827         my ($ds, $drive) = @_;
4828
4829         return if drive_is_cdrom($drive);
4830
4831         my $volid = $drive->{file};
4832
4833         return if !$volid;
4834
4835         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4836
4837         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4838         return if $scfg->{shared};
4839
4840         # replicated disks re-use existing state via bitmap
4841         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
4842         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
4843     });
4844     return $local_volumes;
4845 }
4846
4847 # called in locked context by incoming migration
4848 sub vm_migrate_alloc_nbd_disks {
4849     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
4850
4851     my $format = undef;
4852
4853     my $nbd = {};
4854     foreach my $opt (sort keys %$source_volumes) {
4855         my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
4856
4857         if ($use_existing) {
4858             $nbd->{$opt}->{drivestr} = print_drive($drive);
4859             $nbd->{$opt}->{volid} = $volid;
4860             $nbd->{$opt}->{replicated} = 1;
4861             next;
4862         }
4863
4864         # If a remote storage is specified and the format of the original
4865         # volume is not available there, fall back to the default format.
4866         # Otherwise use the same format as the original.
4867         if (!$storagemap->{identity}) {
4868             $storeid = map_storage($storagemap, $storeid);
4869             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4870             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4871             my $fileFormat = qemu_img_format($scfg, $volname);
4872             $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
4873         } else {
4874             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4875             $format = qemu_img_format($scfg, $volname);
4876         }
4877
4878         my $size = $drive->{size} / 1024;
4879         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
4880         my $newdrive = $drive;
4881         $newdrive->{format} = $format;
4882         $newdrive->{file} = $newvolid;
4883         my $drivestr = print_drive($newdrive);
4884         $nbd->{$opt}->{drivestr} = $drivestr;
4885         $nbd->{$opt}->{volid} = $newvolid;
4886     }
4887
4888     return $nbd;
4889 }
4890
4891 # see vm_start_nolock for parameters, additionally:
4892 # migrate_opts:
4893 #   storagemap = parsed storage map for allocating NBD disks
4894 sub vm_start {
4895     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
4896
4897     return PVE::QemuConfig->lock_config($vmid, sub {
4898         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
4899
4900         die "you can't start a vm if it's a template\n"
4901             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
4902
4903         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
4904
4905         PVE::QemuConfig->check_lock($conf)
4906             if !($params->{skiplock} || $has_suspended_lock);
4907
4908         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
4909
4910         die "VM $vmid already running\n" if check_running($vmid, undef, $migrate_opts->{migratedfrom});
4911
4912         if (my $storagemap = $migrate_opts->{storagemap}) {
4913             my $replicated = $migrate_opts->{replicated_volumes};
4914             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
4915             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
4916
4917             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
4918                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
4919             }
4920         }
4921
4922         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
4923     });
4924 }
4925
4926
4927 # params:
4928 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
4929 #   skiplock => 0/1, skip checking for config lock
4930 #   skiptemplate => 0/1, skip checking whether VM is template
4931 #   forcemachine => to force Qemu machine (rollback/migration)
4932 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
4933 #   timeout => in seconds
4934 #   paused => start VM in paused state (backup)
4935 #   resume => resume from hibernation
4936 # migrate_opts:
4937 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
4938 #   migratedfrom => source node
4939 #   spice_ticket => used for spice migration, passed via tunnel/stdin
4940 #   network => CIDR of migration network
4941 #   type => secure/insecure - tunnel over encrypted connection or plain-text
4942 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
4943 #   replicated_volumes = which volids should be re-used with bitmaps for nbd migration
4944 sub vm_start_nolock {
4945     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
4946
4947     my $statefile = $params->{statefile};
4948     my $resume = $params->{resume};
4949
4950     my $migratedfrom = $migrate_opts->{migratedfrom};
4951     my $migration_type = $migrate_opts->{type};
4952
4953     my $res = {};
4954
4955     # clean up leftover reboot request files
4956     eval { clear_reboot_request($vmid); };
4957     warn $@ if $@;
4958
4959     if (!$statefile && scalar(keys %{$conf->{pending}})) {
4960         vmconfig_apply_pending($vmid, $conf, $storecfg);
4961         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4962     }
4963
4964     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4965
4966     my $defaults = load_defaults();
4967
4968     # set environment variable useful inside network script
4969     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4970
4971     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
4972
4973     my $forcemachine = $params->{forcemachine};
4974     my $forcecpu = $params->{forcecpu};
4975     if ($resume) {
4976         # enforce machine and CPU type on suspended vm to ensure HW compatibility
4977         $forcemachine = $conf->{runningmachine};
4978         $forcecpu = $conf->{runningcpu};
4979         print "Resuming suspended VM\n";
4980     }
4981
4982     my ($cmd, $vollist, $spice_port) =
4983         config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
4984
4985     my $migration_ip;
4986     my $get_migration_ip = sub {
4987         my ($nodename) = @_;
4988
4989         return $migration_ip if defined($migration_ip);
4990
4991         my $cidr = $migrate_opts->{network};
4992
4993         if (!defined($cidr)) {
4994             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4995             $cidr = $dc_conf->{migration}->{network};
4996         }
4997
4998         if (defined($cidr)) {
4999             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5000
5001             die "could not get IP: no address configured on local " .
5002                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5003
5004             die "could not get IP: multiple addresses configured on local " .
5005                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5006
5007             $migration_ip = @$ips[0];
5008         }
5009
5010         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5011             if !defined($migration_ip);
5012
5013         return $migration_ip;
5014     };
5015
5016     my $migrate_uri;
5017     if ($statefile) {
5018         if ($statefile eq 'tcp') {
5019             my $localip = "localhost";
5020             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5021             my $nodename = nodename();
5022
5023             if (!defined($migration_type)) {
5024                 if (defined($datacenterconf->{migration}->{type})) {
5025                     $migration_type = $datacenterconf->{migration}->{type};
5026                 } else {
5027                     $migration_type = 'secure';
5028                 }
5029             }
5030
5031             if ($migration_type eq 'insecure') {
5032                 $localip = $get_migration_ip->($nodename);
5033                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5034             }
5035
5036             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5037             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5038             $migrate_uri = "tcp:${localip}:${migrate_port}";
5039             push @$cmd, '-incoming', $migrate_uri;
5040             push @$cmd, '-S';
5041
5042         } elsif ($statefile eq 'unix') {
5043             # should be default for secure migrations as a ssh TCP forward
5044             # tunnel is not deterministic reliable ready and fails regurarly
5045             # to set up in time, so use UNIX socket forwards
5046             my $socket_addr = "/run/qemu-server/$vmid.migrate";
5047             unlink $socket_addr;
5048
5049             $migrate_uri = "unix:$socket_addr";
5050
5051             push @$cmd, '-incoming', $migrate_uri;
5052             push @$cmd, '-S';
5053
5054         } elsif (-e $statefile) {
5055             push @$cmd, '-loadstate', $statefile;
5056         } else {
5057             my $statepath = PVE::Storage::path($storecfg, $statefile);
5058             push @$vollist, $statefile;
5059             push @$cmd, '-loadstate', $statepath;
5060         }
5061     } elsif ($params->{paused}) {
5062         push @$cmd, '-S';
5063     }
5064
5065     # host pci devices
5066     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5067       my $d = parse_hostpci($conf->{"hostpci$i"});
5068       next if !$d;
5069       my $pcidevices = $d->{pciid};
5070       foreach my $pcidevice (@$pcidevices) {
5071             my $pciid = $pcidevice->{id};
5072
5073             my $info = PVE::SysFSTools::pci_device_info("$pciid");
5074             die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5075             die "no pci device info for device '$pciid'\n" if !$info;
5076
5077             if ($d->{mdev}) {
5078                 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5079                 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5080             } else {
5081                 die "can't unbind/bind pci group to vfio '$pciid'\n"
5082                     if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5083                 die "can't reset pci device '$pciid'\n"
5084                     if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5085             }
5086       }
5087     }
5088
5089     PVE::Storage::activate_volumes($storecfg, $vollist);
5090
5091     eval {
5092         run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5093             outfunc => sub {}, errfunc => sub {});
5094     };
5095     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5096     # timeout should be more than enough here...
5097     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5098
5099     my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5100                                               : $defaults->{cpuunits};
5101
5102     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5103     my %run_params = (
5104         timeout => $statefile ? undef : $start_timeout,
5105         umask => 0077,
5106         noerr => 1,
5107     );
5108
5109     # when migrating, prefix QEMU output so other side can pick up any
5110     # errors that might occur and show the user
5111     if ($migratedfrom) {
5112         $run_params{quiet} = 1;
5113         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5114     }
5115
5116     my %properties = (
5117         Slice => 'qemu.slice',
5118         KillMode => 'none',
5119         CPUShares => $cpuunits
5120     );
5121
5122     if (my $cpulimit = $conf->{cpulimit}) {
5123         $properties{CPUQuota} = int($cpulimit * 100);
5124     }
5125     $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5126
5127     my $run_qemu = sub {
5128         PVE::Tools::run_fork sub {
5129             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5130
5131             my $exitcode = run_command($cmd, %run_params);
5132             die "QEMU exited with code $exitcode\n" if $exitcode;
5133         };
5134     };
5135
5136     if ($conf->{hugepages}) {
5137
5138         my $code = sub {
5139             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5140             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5141
5142             PVE::QemuServer::Memory::hugepages_mount();
5143             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5144
5145             eval { $run_qemu->() };
5146             if (my $err = $@) {
5147                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5148                     if !$conf->{keephugepages};
5149                 die $err;
5150             }
5151
5152             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5153                 if !$conf->{keephugepages};
5154         };
5155         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5156
5157     } else {
5158         eval { $run_qemu->() };
5159     }
5160
5161     if (my $err = $@) {
5162         # deactivate volumes if start fails
5163         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5164         die "start failed: $err";
5165     }
5166
5167     print "migration listens on $migrate_uri\n" if $migrate_uri;
5168     $res->{migrate_uri} = $migrate_uri;
5169
5170     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5171         eval { mon_cmd($vmid, "cont"); };
5172         warn $@ if $@;
5173     }
5174
5175     #start nbd server for storage migration
5176     if (my $nbd = $migrate_opts->{nbd}) {
5177         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5178
5179         my $migrate_storage_uri;
5180         # nbd_protocol_version > 0 for unix socket support
5181         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5182             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5183             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5184             $migrate_storage_uri = "nbd:unix:$socket_path";
5185         } else {
5186             my $nodename = nodename();
5187             my $localip = $get_migration_ip->($nodename);
5188             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5189             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5190
5191             mon_cmd($vmid, "nbd-server-start", addr => {
5192                 type => 'inet',
5193                 data => {
5194                     host => "${localip}",
5195                     port => "${storage_migrate_port}",
5196                 },
5197             });
5198             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5199             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5200         }
5201
5202         $res->{migrate_storage_uri} = $migrate_storage_uri;
5203
5204         foreach my $opt (sort keys %$nbd) {
5205             my $drivestr = $nbd->{$opt}->{drivestr};
5206             my $volid = $nbd->{$opt}->{volid};
5207             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5208             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5209             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5210             print "re-using replicated volume: $opt - $volid\n"
5211                 if $nbd->{$opt}->{replicated};
5212
5213             $res->{drives}->{$opt} = $nbd->{$opt};
5214             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5215         }
5216     }
5217
5218     if ($migratedfrom) {
5219         eval {
5220             set_migration_caps($vmid);
5221         };
5222         warn $@ if $@;
5223
5224         if ($spice_port) {
5225             print "spice listens on port $spice_port\n";
5226             $res->{spice_port} = $spice_port;
5227             if ($migrate_opts->{spice_ticket}) {
5228                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5229                     $migrate_opts->{spice_ticket});
5230                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5231             }
5232         }
5233
5234     } else {
5235         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5236             if !$statefile && $conf->{balloon};
5237
5238         foreach my $opt (keys %$conf) {
5239             next if $opt !~  m/^net\d+$/;
5240             my $nicconf = parse_net($conf->{$opt});
5241             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5242         }
5243     }
5244
5245     mon_cmd($vmid, 'qom-set',
5246                 path => "machine/peripheral/balloon0",
5247                 property => "guest-stats-polling-interval",
5248                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5249
5250     if ($resume) {
5251         print "Resumed VM, removing state\n";
5252         if (my $vmstate = $conf->{vmstate}) {
5253             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5254             PVE::Storage::vdisk_free($storecfg, $vmstate);
5255         }
5256         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5257         PVE::QemuConfig->write_config($vmid, $conf);
5258     }
5259
5260     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5261
5262     return $res;
5263 }
5264
5265 sub vm_commandline {
5266     my ($storecfg, $vmid, $snapname) = @_;
5267
5268     my $conf = PVE::QemuConfig->load_config($vmid);
5269     my $forcemachine;
5270     my $forcecpu;
5271
5272     if ($snapname) {
5273         my $snapshot = $conf->{snapshots}->{$snapname};
5274         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5275
5276         # check for machine or CPU overrides in snapshot
5277         $forcemachine = $snapshot->{runningmachine};
5278         $forcecpu = $snapshot->{runningcpu};
5279
5280         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5281
5282         $conf = $snapshot;
5283     }
5284
5285     my $defaults = load_defaults();
5286
5287     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults,
5288         $forcemachine, $forcecpu);
5289
5290     return PVE::Tools::cmd2string($cmd);
5291 }
5292
5293 sub vm_reset {
5294     my ($vmid, $skiplock) = @_;
5295
5296     PVE::QemuConfig->lock_config($vmid, sub {
5297
5298         my $conf = PVE::QemuConfig->load_config($vmid);
5299
5300         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5301
5302         mon_cmd($vmid, "system_reset");
5303     });
5304 }
5305
5306 sub get_vm_volumes {
5307     my ($conf) = @_;
5308
5309     my $vollist = [];
5310     foreach_volid($conf, sub {
5311         my ($volid, $attr) = @_;
5312
5313         return if $volid =~ m|^/|;
5314
5315         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5316         return if !$sid;
5317
5318         push @$vollist, $volid;
5319     });
5320
5321     return $vollist;
5322 }
5323
5324 sub vm_stop_cleanup {
5325     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5326
5327     eval {
5328
5329         if (!$keepActive) {
5330             my $vollist = get_vm_volumes($conf);
5331             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5332         }
5333
5334         foreach my $ext (qw(mon qmp pid vnc qga)) {
5335             unlink "/var/run/qemu-server/${vmid}.$ext";
5336         }
5337
5338         if ($conf->{ivshmem}) {
5339             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5340             # just delete it for now, VMs which have this already open do not
5341             # are affected, but new VMs will get a separated one. If this
5342             # becomes an issue we either add some sort of ref-counting or just
5343             # add a "don't delete on stop" flag to the ivshmem format.
5344             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5345         }
5346
5347         foreach my $key (keys %$conf) {
5348             next if $key !~ m/^hostpci(\d+)$/;
5349             my $hostpciindex = $1;
5350             my $d = parse_hostpci($conf->{$key});
5351             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5352
5353             foreach my $pci (@{$d->{pciid}}) {
5354                 my $pciid = $pci->{id};
5355                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5356             }
5357         }
5358
5359         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5360     };
5361     warn $@ if $@; # avoid errors - just warn
5362 }
5363
5364 # call only in locked context
5365 sub _do_vm_stop {
5366     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5367
5368     my $pid = check_running($vmid, $nocheck);
5369     return if !$pid;
5370
5371     my $conf;
5372     if (!$nocheck) {
5373         $conf = PVE::QemuConfig->load_config($vmid);
5374         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5375         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5376             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5377             $timeout = $opts->{down} if $opts->{down};
5378         }
5379         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5380     }
5381
5382     eval {
5383         if ($shutdown) {
5384             if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5385                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5386             } else {
5387                 mon_cmd($vmid, "system_powerdown");
5388             }
5389         } else {
5390             mon_cmd($vmid, "quit");
5391         }
5392     };
5393     my $err = $@;
5394
5395     if (!$err) {
5396         $timeout = 60 if !defined($timeout);
5397
5398         my $count = 0;
5399         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5400             $count++;
5401             sleep 1;
5402         }
5403
5404         if ($count >= $timeout) {
5405             if ($force) {
5406                 warn "VM still running - terminating now with SIGTERM\n";
5407                 kill 15, $pid;
5408             } else {
5409                 die "VM quit/powerdown failed - got timeout\n";
5410             }
5411         } else {
5412             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5413             return;
5414         }
5415     } else {
5416        if (!check_running($vmid, $nocheck)) {
5417             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5418             return;
5419        }
5420        if ($force) {
5421             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5422             kill 15, $pid;
5423         } else {
5424             die "VM quit/powerdown failed\n";
5425         }
5426     }
5427
5428     # wait again
5429     $timeout = 10;
5430
5431     my $count = 0;
5432     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5433         $count++;
5434         sleep 1;
5435     }
5436
5437     if ($count >= $timeout) {
5438         warn "VM still running - terminating now with SIGKILL\n";
5439         kill 9, $pid;
5440         sleep 1;
5441     }
5442
5443     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5444 }
5445
5446 # Note: use $nocheck to skip tests if VM configuration file exists.
5447 # We need that when migration VMs to other nodes (files already moved)
5448 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5449 sub vm_stop {
5450     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5451
5452     $force = 1 if !defined($force) && !$shutdown;
5453
5454     if ($migratedfrom){
5455         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5456         kill 15, $pid if $pid;
5457         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5458         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5459         return;
5460     }
5461
5462     PVE::QemuConfig->lock_config($vmid, sub {
5463         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5464    });
5465 }
5466
5467 sub vm_reboot {
5468     my ($vmid, $timeout) = @_;
5469
5470     PVE::QemuConfig->lock_config($vmid, sub {
5471         eval {
5472
5473             # only reboot if running, as qmeventd starts it again on a stop event
5474             return if !check_running($vmid);
5475
5476             create_reboot_request($vmid);
5477
5478             my $storecfg = PVE::Storage::config();
5479             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5480
5481         };
5482         if (my $err = $@) {
5483             # avoid that the next normal shutdown will be confused for a reboot
5484             clear_reboot_request($vmid);
5485             die $err;
5486         }
5487    });
5488 }
5489
5490 # note: if using the statestorage parameter, the caller has to check privileges
5491 sub vm_suspend {
5492     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5493
5494     my $conf;
5495     my $path;
5496     my $storecfg;
5497     my $vmstate;
5498
5499     PVE::QemuConfig->lock_config($vmid, sub {
5500
5501         $conf = PVE::QemuConfig->load_config($vmid);
5502
5503         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5504         PVE::QemuConfig->check_lock($conf)
5505             if !($skiplock || $is_backing_up);
5506
5507         die "cannot suspend to disk during backup\n"
5508             if $is_backing_up && $includestate;
5509
5510         if ($includestate) {
5511             $conf->{lock} = 'suspending';
5512             my $date = strftime("%Y-%m-%d", localtime(time()));
5513             $storecfg = PVE::Storage::config();
5514             if (!$statestorage) {
5515                 $statestorage = find_vmstate_storage($conf, $storecfg);
5516                 # check permissions for the storage
5517                 my $rpcenv = PVE::RPCEnvironment::get();
5518                 if ($rpcenv->{type} ne 'cli') {
5519                     my $authuser = $rpcenv->get_user();
5520                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5521                 }
5522             }
5523
5524
5525             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
5526                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5527             $path = PVE::Storage::path($storecfg, $vmstate);
5528             PVE::QemuConfig->write_config($vmid, $conf);
5529         } else {
5530             mon_cmd($vmid, "stop");
5531         }
5532     });
5533
5534     if ($includestate) {
5535         # save vm state
5536         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5537
5538         eval {
5539             mon_cmd($vmid, "savevm-start", statefile => $path);
5540             for(;;) {
5541                 my $state = mon_cmd($vmid, "query-savevm");
5542                 if (!$state->{status}) {
5543                     die "savevm not active\n";
5544                 } elsif ($state->{status} eq 'active') {
5545                     sleep(1);
5546                     next;
5547                 } elsif ($state->{status} eq 'completed') {
5548                     print "State saved, quitting\n";
5549                     last;
5550                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5551                     die "query-savevm failed with error '$state->{error}'\n"
5552                 } else {
5553                     die "query-savevm returned status '$state->{status}'\n";
5554                 }
5555             }
5556         };
5557         my $err = $@;
5558
5559         PVE::QemuConfig->lock_config($vmid, sub {
5560             $conf = PVE::QemuConfig->load_config($vmid);
5561             if ($err) {
5562                 # cleanup, but leave suspending lock, to indicate something went wrong
5563                 eval {
5564                     mon_cmd($vmid, "savevm-end");
5565                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5566                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5567                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
5568                     PVE::QemuConfig->write_config($vmid, $conf);
5569                 };
5570                 warn $@ if $@;
5571                 die $err;
5572             }
5573
5574             die "lock changed unexpectedly\n"
5575                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5576
5577             mon_cmd($vmid, "quit");
5578             $conf->{lock} = 'suspended';
5579             PVE::QemuConfig->write_config($vmid, $conf);
5580         });
5581     }
5582 }
5583
5584 sub vm_resume {
5585     my ($vmid, $skiplock, $nocheck) = @_;
5586
5587     PVE::QemuConfig->lock_config($vmid, sub {
5588         my $res = mon_cmd($vmid, 'query-status');
5589         my $resume_cmd = 'cont';
5590
5591         if ($res->{status} && $res->{status} eq 'suspended') {
5592             $resume_cmd = 'system_wakeup';
5593         }
5594
5595         if (!$nocheck) {
5596
5597             my $conf = PVE::QemuConfig->load_config($vmid);
5598
5599             PVE::QemuConfig->check_lock($conf)
5600                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5601         }
5602
5603         mon_cmd($vmid, $resume_cmd);
5604     });
5605 }
5606
5607 sub vm_sendkey {
5608     my ($vmid, $skiplock, $key) = @_;
5609
5610     PVE::QemuConfig->lock_config($vmid, sub {
5611
5612         my $conf = PVE::QemuConfig->load_config($vmid);
5613
5614         # there is no qmp command, so we use the human monitor command
5615         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5616         die $res if $res ne '';
5617     });
5618 }
5619
5620 # vzdump restore implementaion
5621
5622 sub tar_archive_read_firstfile {
5623     my $archive = shift;
5624
5625     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5626
5627     # try to detect archive type first
5628     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5629         die "unable to open file '$archive'\n";
5630     my $firstfile = <$fh>;
5631     kill 15, $pid;
5632     close $fh;
5633
5634     die "ERROR: archive contaions no data\n" if !$firstfile;
5635     chomp $firstfile;
5636
5637     return $firstfile;
5638 }
5639
5640 sub tar_restore_cleanup {
5641     my ($storecfg, $statfile) = @_;
5642
5643     print STDERR "starting cleanup\n";
5644
5645     if (my $fd = IO::File->new($statfile, "r")) {
5646         while (defined(my $line = <$fd>)) {
5647             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5648                 my $volid = $2;
5649                 eval {
5650                     if ($volid =~ m|^/|) {
5651                         unlink $volid || die 'unlink failed\n';
5652                     } else {
5653                         PVE::Storage::vdisk_free($storecfg, $volid);
5654                     }
5655                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5656                 };
5657                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5658             } else {
5659                 print STDERR "unable to parse line in statfile - $line";
5660             }
5661         }
5662         $fd->close();
5663     }
5664 }
5665
5666 sub restore_file_archive {
5667     my ($archive, $vmid, $user, $opts) = @_;
5668
5669     return restore_vma_archive($archive, $vmid, $user, $opts)
5670         if $archive eq '-';
5671
5672     my $info = PVE::Storage::archive_info($archive);
5673     my $format = $opts->{format} // $info->{format};
5674     my $comp = $info->{compression};
5675
5676     # try to detect archive format
5677     if ($format eq 'tar') {
5678         return restore_tar_archive($archive, $vmid, $user, $opts);
5679     } else {
5680         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5681     }
5682 }
5683
5684 # hepler to remove disks that will not be used after restore
5685 my $restore_cleanup_oldconf = sub {
5686     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
5687
5688     PVE::QemuConfig->foreach_volume($oldconf, sub {
5689         my ($ds, $drive) = @_;
5690
5691         return if drive_is_cdrom($drive, 1);
5692
5693         my $volid = $drive->{file};
5694         return if !$volid || $volid =~ m|^/|;
5695
5696         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
5697         return if !$path || !$owner || ($owner != $vmid);
5698
5699         # Note: only delete disk we want to restore
5700         # other volumes will become unused
5701         if ($virtdev_hash->{$ds}) {
5702             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
5703             if (my $err = $@) {
5704                 warn $err;
5705             }
5706         }
5707     });
5708
5709     # delete vmstate files, after the restore we have no snapshots anymore
5710     foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5711         my $snap = $oldconf->{snapshots}->{$snapname};
5712         if ($snap->{vmstate}) {
5713             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
5714             if (my $err = $@) {
5715                 warn $err;
5716             }
5717         }
5718     }
5719 };
5720
5721 # Helper to parse vzdump backup device hints
5722 #
5723 # $rpcenv: Environment, used to ckeck storage permissions
5724 # $user: User ID, to check storage permissions
5725 # $storecfg: Storage configuration
5726 # $fh: the file handle for reading the configuration
5727 # $devinfo: should contain device sizes for all backu-up'ed devices
5728 # $options: backup options (pool, default storage)
5729 #
5730 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
5731 my $parse_backup_hints = sub {
5732     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
5733
5734     my $virtdev_hash = {};
5735
5736     while (defined(my $line = <$fh>)) {
5737         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5738             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5739             die "archive does not contain data for drive '$virtdev'\n"
5740                 if !$devinfo->{$devname};
5741
5742             if (defined($options->{storage})) {
5743                 $storeid = $options->{storage} || 'local';
5744             } elsif (!$storeid) {
5745                 $storeid = 'local';
5746             }
5747             $format = 'raw' if !$format;
5748             $devinfo->{$devname}->{devname} = $devname;
5749             $devinfo->{$devname}->{virtdev} = $virtdev;
5750             $devinfo->{$devname}->{format} = $format;
5751             $devinfo->{$devname}->{storeid} = $storeid;
5752
5753             # check permission on storage
5754             my $pool = $options->{pool}; # todo: do we need that?
5755             if ($user ne 'root@pam') {
5756                 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5757             }
5758
5759             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5760         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
5761             my $virtdev = $1;
5762             my $drive = parse_drive($virtdev, $2);
5763             if (drive_is_cloudinit($drive)) {
5764                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5765                 $storeid = $options->{storage} if defined ($options->{storage});
5766                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5767                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
5768
5769                 $virtdev_hash->{$virtdev} = {
5770                     format => $format,
5771                     storeid => $storeid,
5772                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
5773                     is_cloudinit => 1,
5774                 };
5775             }
5776         }
5777     }
5778
5779     return $virtdev_hash;
5780 };
5781
5782 # Helper to allocate and activate all volumes required for a restore
5783 #
5784 # $storecfg: Storage configuration
5785 # $virtdev_hash: as returned by parse_backup_hints()
5786 #
5787 # Returns: { $virtdev => $volid }
5788 my $restore_allocate_devices = sub {
5789     my ($storecfg, $virtdev_hash, $vmid) = @_;
5790
5791     my $map = {};
5792     foreach my $virtdev (sort keys %$virtdev_hash) {
5793         my $d = $virtdev_hash->{$virtdev};
5794         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5795         my $storeid = $d->{storeid};
5796         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5797
5798         # test if requested format is supported
5799         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5800         my $supported = grep { $_ eq $d->{format} } @$validFormats;
5801         $d->{format} = $defFormat if !$supported;
5802
5803         my $name;
5804         if ($d->{is_cloudinit}) {
5805             $name = "vm-$vmid-cloudinit";
5806             $name .= ".$d->{format}" if $d->{format} ne 'raw';
5807         }
5808
5809         my $volid = PVE::Storage::vdisk_alloc(
5810             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
5811
5812         print STDERR "new volume ID is '$volid'\n";
5813         $d->{volid} = $volid;
5814
5815         PVE::Storage::activate_volumes($storecfg, [$volid]);
5816
5817         $map->{$virtdev} = $volid;
5818     }
5819
5820     return $map;
5821 };
5822
5823 my $restore_update_config_line = sub {
5824     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5825
5826     return if $line =~ m/^\#qmdump\#/;
5827     return if $line =~ m/^\#vzdump\#/;
5828     return if $line =~ m/^lock:/;
5829     return if $line =~ m/^unused\d+:/;
5830     return if $line =~ m/^parent:/;
5831
5832     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5833     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5834         # try to convert old 1.X settings
5835         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5836         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5837             my ($model, $macaddr) = split(/\=/, $devconfig);
5838             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5839             my $net = {
5840                 model => $model,
5841                 bridge => "vmbr$ind",
5842                 macaddr => $macaddr,
5843             };
5844             my $netstr = print_net($net);
5845
5846             print $outfd "net$cookie->{netcount}: $netstr\n";
5847             $cookie->{netcount}++;
5848         }
5849     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5850         my ($id, $netstr) = ($1, $2);
5851         my $net = parse_net($netstr);
5852         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5853         $netstr = print_net($net);
5854         print $outfd "$id: $netstr\n";
5855     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5856         my $virtdev = $1;
5857         my $value = $3;
5858         my $di = parse_drive($virtdev, $value);
5859         if (defined($di->{backup}) && !$di->{backup}) {
5860             print $outfd "#$line";
5861         } elsif ($map->{$virtdev}) {
5862             delete $di->{format}; # format can change on restore
5863             $di->{file} = $map->{$virtdev};
5864             $value = print_drive($di);
5865             print $outfd "$virtdev: $value\n";
5866         } else {
5867             print $outfd $line;
5868         }
5869     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5870         my $vmgenid = $1;
5871         if ($vmgenid ne '0') {
5872             # always generate a new vmgenid if there was a valid one setup
5873             $vmgenid = generate_uuid();
5874         }
5875         print $outfd "vmgenid: $vmgenid\n";
5876     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5877         my ($uuid, $uuid_str);
5878         UUID::generate($uuid);
5879         UUID::unparse($uuid, $uuid_str);
5880         my $smbios1 = parse_smbios1($2);
5881         $smbios1->{uuid} = $uuid_str;
5882         print $outfd $1.print_smbios1($smbios1)."\n";
5883     } else {
5884         print $outfd $line;
5885     }
5886 };
5887
5888 my $restore_deactivate_volumes = sub {
5889     my ($storecfg, $devinfo) = @_;
5890
5891     my $vollist = [];
5892     foreach my $devname (keys %$devinfo) {
5893         my $volid = $devinfo->{$devname}->{volid};
5894         push @$vollist, $volid if $volid;
5895     }
5896
5897     PVE::Storage::deactivate_volumes($storecfg, $vollist);
5898 };
5899
5900 my $restore_destroy_volumes = sub {
5901     my ($storecfg, $devinfo) = @_;
5902
5903     foreach my $devname (keys %$devinfo) {
5904         my $volid = $devinfo->{$devname}->{volid};
5905         next if !$volid;
5906         eval {
5907             if ($volid =~ m|^/|) {
5908                 unlink $volid || die 'unlink failed\n';
5909             } else {
5910                 PVE::Storage::vdisk_free($storecfg, $volid);
5911             }
5912             print STDERR "temporary volume '$volid' sucessfuly removed\n";
5913         };
5914         print STDERR "unable to cleanup '$volid' - $@" if $@;
5915     }
5916 };
5917
5918 sub scan_volids {
5919     my ($cfg, $vmid) = @_;
5920
5921     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5922
5923     my $volid_hash = {};
5924     foreach my $storeid (keys %$info) {
5925         foreach my $item (@{$info->{$storeid}}) {
5926             next if !($item->{volid} && $item->{size});
5927             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5928             $volid_hash->{$item->{volid}} = $item;
5929         }
5930     }
5931
5932     return $volid_hash;
5933 }
5934
5935 sub update_disk_config {
5936     my ($vmid, $conf, $volid_hash) = @_;
5937
5938     my $changes;
5939     my $prefix = "VM $vmid";
5940
5941     # used and unused disks
5942     my $referenced = {};
5943
5944     # Note: it is allowed to define multiple storages with same path (alias), so
5945     # we need to check both 'volid' and real 'path' (two different volid can point
5946     # to the same path).
5947
5948     my $referencedpath = {};
5949
5950     # update size info
5951     PVE::QemuConfig->foreach_volume($conf, sub {
5952         my ($opt, $drive) = @_;
5953
5954         my $volid = $drive->{file};
5955         return if !$volid;
5956         my $volume = $volid_hash->{$volid};
5957
5958         # mark volid as "in-use" for next step
5959         $referenced->{$volid} = 1;
5960         if ($volume && (my $path = $volume->{path})) {
5961             $referencedpath->{$path} = 1;
5962         }
5963
5964         return if drive_is_cdrom($drive);
5965         return if !$volume;
5966
5967         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
5968         if (defined($updated)) {
5969             $changes = 1;
5970             $conf->{$opt} = print_drive($updated);
5971             print "$prefix ($opt): $msg\n";
5972         }
5973     });
5974
5975     # remove 'unusedX' entry if volume is used
5976     PVE::QemuConfig->foreach_unused_volume($conf, sub {
5977         my ($opt, $drive) = @_;
5978
5979         my $volid = $drive->{file};
5980         return if !$volid;
5981
5982         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5983         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5984             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
5985             $changes = 1;
5986             delete $conf->{$opt};
5987         }
5988
5989         $referenced->{$volid} = 1;
5990         $referencedpath->{$path} = 1 if $path;
5991     });
5992
5993     foreach my $volid (sort keys %$volid_hash) {
5994         next if $volid =~ m/vm-$vmid-state-/;
5995         next if $referenced->{$volid};
5996         my $path = $volid_hash->{$volid}->{path};
5997         next if !$path; # just to be sure
5998         next if $referencedpath->{$path};
5999         $changes = 1;
6000         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6001         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6002         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6003     }
6004
6005     return $changes;
6006 }
6007
6008 sub rescan {
6009     my ($vmid, $nolock, $dryrun) = @_;
6010
6011     my $cfg = PVE::Storage::config();
6012
6013     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6014     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6015     foreach my $stor (keys %{$cfg->{ids}}) {
6016         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6017     }
6018
6019     print "rescan volumes...\n";
6020     my $volid_hash = scan_volids($cfg, $vmid);
6021
6022     my $updatefn =  sub {
6023         my ($vmid) = @_;
6024
6025         my $conf = PVE::QemuConfig->load_config($vmid);
6026
6027         PVE::QemuConfig->check_lock($conf);
6028
6029         my $vm_volids = {};
6030         foreach my $volid (keys %$volid_hash) {
6031             my $info = $volid_hash->{$volid};
6032             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6033         }
6034
6035         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6036
6037         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6038     };
6039
6040     if (defined($vmid)) {
6041         if ($nolock) {
6042             &$updatefn($vmid);
6043         } else {
6044             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6045         }
6046     } else {
6047         my $vmlist = config_list();
6048         foreach my $vmid (keys %$vmlist) {
6049             if ($nolock) {
6050                 &$updatefn($vmid);
6051             } else {
6052                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6053             }
6054         }
6055     }
6056 }
6057
6058 sub restore_proxmox_backup_archive {
6059     my ($archive, $vmid, $user, $options) = @_;
6060
6061     my $storecfg = PVE::Storage::config();
6062
6063     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6064     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6065
6066     my $server = $scfg->{server};
6067     my $datastore = $scfg->{datastore};
6068     my $username = $scfg->{username} // 'root@pam';
6069     my $fingerprint = $scfg->{fingerprint};
6070     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6071
6072     my $repo = "$username\@$server:$datastore";
6073
6074     # This is only used for `pbs-restore`!
6075     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6076     local $ENV{PBS_PASSWORD} = $password;
6077     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6078
6079     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6080         PVE::Storage::parse_volname($storecfg, $archive);
6081
6082     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6083
6084     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6085
6086     my $tmpdir = "/var/tmp/vzdumptmp$$";
6087     rmtree $tmpdir;
6088     mkpath $tmpdir;
6089
6090     my $conffile = PVE::QemuConfig->config_file($vmid);
6091     my $tmpfn = "$conffile.$$.tmp";
6092      # disable interrupts (always do cleanups)
6093     local $SIG{INT} =
6094         local $SIG{TERM} =
6095         local $SIG{QUIT} =
6096         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6097
6098     # Note: $oldconf is undef if VM does not exists
6099     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6100     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6101
6102     my $rpcenv = PVE::RPCEnvironment::get();
6103     my $devinfo = {};
6104
6105     eval {
6106         # enable interrupts
6107         local $SIG{INT} =
6108             local $SIG{TERM} =
6109             local $SIG{QUIT} =
6110             local $SIG{HUP} =
6111             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6112
6113         my $cfgfn = "$tmpdir/qemu-server.conf";
6114         my $firewall_config_fn = "$tmpdir/fw.conf";
6115         my $index_fn = "$tmpdir/index.json";
6116
6117         my $cmd = "restore";
6118
6119         my $param = [$pbs_backup_name, "index.json", $index_fn];
6120         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6121         my $index = PVE::Tools::file_get_contents($index_fn);
6122         $index = decode_json($index);
6123
6124         # print Dumper($index);
6125         foreach my $info (@{$index->{files}}) {
6126             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6127                 my $devname = $1;
6128                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6129                     $devinfo->{$devname}->{size} = $1;
6130                 } else {
6131                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6132                 }
6133             }
6134         }
6135
6136         my $is_qemu_server_backup = scalar(
6137             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6138         );
6139         if (!$is_qemu_server_backup) {
6140             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6141         }
6142         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6143
6144         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6145         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6146
6147         if ($has_firewall_config) {
6148             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6149             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6150
6151             my $pve_firewall_dir = '/etc/pve/firewall';
6152             mkdir $pve_firewall_dir; # make sure the dir exists
6153             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6154         }
6155
6156         my $fh = IO::File->new($cfgfn, "r") ||
6157             die "unable to read qemu-server.conf - $!\n";
6158
6159         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6160
6161         # fixme: rate limit?
6162
6163         # create empty/temp config
6164         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6165
6166         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6167
6168         # allocate volumes
6169         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6170
6171         foreach my $virtdev (sort keys %$virtdev_hash) {
6172             my $d = $virtdev_hash->{$virtdev};
6173             next if $d->{is_cloudinit}; # no need to restore cloudinit
6174
6175             my $volid = $d->{volid};
6176
6177             my $path = PVE::Storage::path($storecfg, $volid);
6178
6179             # This is the ONLY user of the PBS_ env vars set on top of this function!
6180             my $pbs_restore_cmd = [
6181                 '/usr/bin/pbs-restore',
6182                 '--repository', $repo,
6183                 $pbs_backup_name,
6184                 "$d->{devname}.img.fidx",
6185                 $path,
6186                 '--verbose',
6187                 ];
6188
6189             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6190             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
6191
6192             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6193                 push @$pbs_restore_cmd, '--skip-zero';
6194             }
6195
6196             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6197             print "restore proxmox backup image: $dbg_cmdstring\n";
6198             run_command($pbs_restore_cmd);
6199         }
6200
6201         $fh->seek(0, 0) || die "seek failed - $!\n";
6202
6203         my $outfd = new IO::File ($tmpfn, "w") ||
6204             die "unable to write config for VM $vmid\n";
6205
6206         my $cookie = { netcount => 0 };
6207         while (defined(my $line = <$fh>)) {
6208             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $options->{unique});
6209         }
6210
6211         $fh->close();
6212         $outfd->close();
6213     };
6214     my $err = $@;
6215
6216     $restore_deactivate_volumes->($storecfg, $devinfo);
6217
6218     rmtree $tmpdir;
6219
6220     if ($err) {
6221         unlink $tmpfn;
6222         $restore_destroy_volumes->($storecfg, $devinfo);
6223         die $err;
6224     }
6225
6226     rename($tmpfn, $conffile) ||
6227         die "unable to commit configuration file '$conffile'\n";
6228
6229     PVE::Cluster::cfs_update(); # make sure we read new file
6230
6231     eval { rescan($vmid, 1); };
6232     warn $@ if $@;
6233 }
6234
6235 sub restore_vma_archive {
6236     my ($archive, $vmid, $user, $opts, $comp) = @_;
6237
6238     my $readfrom = $archive;
6239
6240     my $cfg = PVE::Storage::config();
6241     my $commands = [];
6242     my $bwlimit = $opts->{bwlimit};
6243
6244     my $dbg_cmdstring = '';
6245     my $add_pipe = sub {
6246         my ($cmd) = @_;
6247         push @$commands, $cmd;
6248         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6249         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6250         $readfrom = '-';
6251     };
6252
6253     my $input = undef;
6254     if ($archive eq '-') {
6255         $input = '<&STDIN';
6256     } else {
6257         # If we use a backup from a PVE defined storage we also consider that
6258         # storage's rate limit:
6259         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6260         if (defined($volid)) {
6261             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6262             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6263             if ($readlimit) {
6264                 print STDERR "applying read rate limit: $readlimit\n";
6265                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6266                 $add_pipe->($cstream);
6267             }
6268         }
6269     }
6270
6271     if ($comp) {
6272         my $info = PVE::Storage::decompressor_info('vma', $comp);
6273         my $cmd = $info->{decompressor};
6274         push @$cmd, $readfrom;
6275         $add_pipe->($cmd);
6276     }
6277
6278     my $tmpdir = "/var/tmp/vzdumptmp$$";
6279     rmtree $tmpdir;
6280
6281     # disable interrupts (always do cleanups)
6282     local $SIG{INT} =
6283         local $SIG{TERM} =
6284         local $SIG{QUIT} =
6285         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6286
6287     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6288     POSIX::mkfifo($mapfifo, 0600);
6289     my $fifofh;
6290
6291     my $openfifo = sub {
6292         open($fifofh, '>', $mapfifo) || die $!;
6293     };
6294
6295     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6296
6297     my $oldtimeout;
6298     my $timeout = 5;
6299
6300     my $devinfo = {};
6301
6302     my $rpcenv = PVE::RPCEnvironment::get();
6303
6304     my $conffile = PVE::QemuConfig->config_file($vmid);
6305     my $tmpfn = "$conffile.$$.tmp";
6306
6307     # Note: $oldconf is undef if VM does not exist
6308     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6309     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6310
6311     my %storage_limits;
6312
6313     my $print_devmap = sub {
6314         my $cfgfn = "$tmpdir/qemu-server.conf";
6315
6316         # we can read the config - that is already extracted
6317         my $fh = IO::File->new($cfgfn, "r") ||
6318             die "unable to read qemu-server.conf - $!\n";
6319
6320         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6321         if (-f $fwcfgfn) {
6322             my $pve_firewall_dir = '/etc/pve/firewall';
6323             mkdir $pve_firewall_dir; # make sure the dir exists
6324             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6325         }
6326
6327         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6328
6329         foreach my $key (keys %storage_limits) {
6330             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6331             next if !$limit;
6332             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6333             $storage_limits{$key} = $limit * 1024;
6334         }
6335
6336         foreach my $devname (keys %$devinfo) {
6337             die "found no device mapping information for device '$devname'\n"
6338                 if !$devinfo->{$devname}->{virtdev};
6339         }
6340
6341         # create empty/temp config
6342         if ($oldconf) {
6343             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6344             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6345         }
6346
6347         # allocate volumes
6348         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6349
6350         # print restore information to $fifofh
6351         foreach my $virtdev (sort keys %$virtdev_hash) {
6352             my $d = $virtdev_hash->{$virtdev};
6353             next if $d->{is_cloudinit}; # no need to restore cloudinit
6354
6355             my $storeid = $d->{storeid};
6356             my $volid = $d->{volid};
6357
6358             my $map_opts = '';
6359             if (my $limit = $storage_limits{$storeid}) {
6360                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6361             }
6362
6363             my $write_zeros = 1;
6364             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6365                 $write_zeros = 0;
6366             }
6367
6368             my $path = PVE::Storage::path($cfg, $volid);
6369
6370             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6371
6372             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6373         }
6374
6375         $fh->seek(0, 0) || die "seek failed - $!\n";
6376
6377         my $outfd = new IO::File ($tmpfn, "w") ||
6378             die "unable to write config for VM $vmid\n";
6379
6380         my $cookie = { netcount => 0 };
6381         while (defined(my $line = <$fh>)) {
6382             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6383         }
6384
6385         $fh->close();
6386         $outfd->close();
6387     };
6388
6389     eval {
6390         # enable interrupts
6391         local $SIG{INT} =
6392             local $SIG{TERM} =
6393             local $SIG{QUIT} =
6394             local $SIG{HUP} =
6395             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6396         local $SIG{ALRM} = sub { die "got timeout\n"; };
6397
6398         $oldtimeout = alarm($timeout);
6399
6400         my $parser = sub {
6401             my $line = shift;
6402
6403             print "$line\n";
6404
6405             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6406                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6407                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6408             } elsif ($line =~ m/^CTIME: /) {
6409                 # we correctly received the vma config, so we can disable
6410                 # the timeout now for disk allocation (set to 10 minutes, so
6411                 # that we always timeout if something goes wrong)
6412                 alarm(600);
6413                 &$print_devmap();
6414                 print $fifofh "done\n";
6415                 my $tmp = $oldtimeout || 0;
6416                 $oldtimeout = undef;
6417                 alarm($tmp);
6418                 close($fifofh);
6419             }
6420         };
6421
6422         print "restore vma archive: $dbg_cmdstring\n";
6423         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6424     };
6425     my $err = $@;
6426
6427     alarm($oldtimeout) if $oldtimeout;
6428
6429     $restore_deactivate_volumes->($cfg, $devinfo);
6430
6431     unlink $mapfifo;
6432     rmtree $tmpdir;
6433
6434     if ($err) {
6435         unlink $tmpfn;
6436         $restore_destroy_volumes->($cfg, $devinfo);
6437         die $err;
6438     }
6439
6440     rename($tmpfn, $conffile) ||
6441         die "unable to commit configuration file '$conffile'\n";
6442
6443     PVE::Cluster::cfs_update(); # make sure we read new file
6444
6445     eval { rescan($vmid, 1); };
6446     warn $@ if $@;
6447 }
6448
6449 sub restore_tar_archive {
6450     my ($archive, $vmid, $user, $opts) = @_;
6451
6452     if ($archive ne '-') {
6453         my $firstfile = tar_archive_read_firstfile($archive);
6454         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
6455             if $firstfile ne 'qemu-server.conf';
6456     }
6457
6458     my $storecfg = PVE::Storage::config();
6459
6460     # avoid zombie disks when restoring over an existing VM -> cleanup first
6461     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6462     # skiplock=1 because qmrestore has set the 'create' lock itself already
6463     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6464     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6465
6466     my $tocmd = "/usr/lib/qemu-server/qmextract";
6467
6468     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6469     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6470     $tocmd .= ' --prealloc' if $opts->{prealloc};
6471     $tocmd .= ' --info' if $opts->{info};
6472
6473     # tar option "xf" does not autodetect compression when read from STDIN,
6474     # so we pipe to zcat
6475     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6476         PVE::Tools::shellquote("--to-command=$tocmd");
6477
6478     my $tmpdir = "/var/tmp/vzdumptmp$$";
6479     mkpath $tmpdir;
6480
6481     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6482     local $ENV{VZDUMP_VMID} = $vmid;
6483     local $ENV{VZDUMP_USER} = $user;
6484
6485     my $conffile = PVE::QemuConfig->config_file($vmid);
6486     my $tmpfn = "$conffile.$$.tmp";
6487
6488     # disable interrupts (always do cleanups)
6489     local $SIG{INT} =
6490         local $SIG{TERM} =
6491         local $SIG{QUIT} =
6492         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6493
6494     eval {
6495         # enable interrupts
6496         local $SIG{INT} =
6497             local $SIG{TERM} =
6498             local $SIG{QUIT} =
6499             local $SIG{HUP} =
6500             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6501
6502         if ($archive eq '-') {
6503             print "extracting archive from STDIN\n";
6504             run_command($cmd, input => "<&STDIN");
6505         } else {
6506             print "extracting archive '$archive'\n";
6507             run_command($cmd);
6508         }
6509
6510         return if $opts->{info};
6511
6512         # read new mapping
6513         my $map = {};
6514         my $statfile = "$tmpdir/qmrestore.stat";
6515         if (my $fd = IO::File->new($statfile, "r")) {
6516             while (defined (my $line = <$fd>)) {
6517                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6518                     $map->{$1} = $2 if $1;
6519                 } else {
6520                     print STDERR "unable to parse line in statfile - $line\n";
6521                 }
6522             }
6523             $fd->close();
6524         }
6525
6526         my $confsrc = "$tmpdir/qemu-server.conf";
6527
6528         my $srcfd = new IO::File($confsrc, "r") ||
6529             die "unable to open file '$confsrc'\n";
6530
6531         my $outfd = new IO::File ($tmpfn, "w") ||
6532             die "unable to write config for VM $vmid\n";
6533
6534         my $cookie = { netcount => 0 };
6535         while (defined (my $line = <$srcfd>)) {
6536             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6537         }
6538
6539         $srcfd->close();
6540         $outfd->close();
6541     };
6542     if (my $err = $@) {
6543         unlink $tmpfn;
6544         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6545         die $err;
6546     }
6547
6548     rmtree $tmpdir;
6549
6550     rename $tmpfn, $conffile ||
6551         die "unable to commit configuration file '$conffile'\n";
6552
6553     PVE::Cluster::cfs_update(); # make sure we read new file
6554
6555     eval { rescan($vmid, 1); };
6556     warn $@ if $@;
6557 };
6558
6559 sub foreach_storage_used_by_vm {
6560     my ($conf, $func) = @_;
6561
6562     my $sidhash = {};
6563
6564     PVE::QemuConfig->foreach_volume($conf, sub {
6565         my ($ds, $drive) = @_;
6566         return if drive_is_cdrom($drive);
6567
6568         my $volid = $drive->{file};
6569
6570         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6571         $sidhash->{$sid} = $sid if $sid;
6572     });
6573
6574     foreach my $sid (sort keys %$sidhash) {
6575         &$func($sid);
6576     }
6577 }
6578
6579 my $qemu_snap_storage = {
6580     rbd => 1,
6581 };
6582 sub do_snapshots_with_qemu {
6583     my ($storecfg, $volid) = @_;
6584
6585     my $storage_name = PVE::Storage::parse_volume_id($volid);
6586     my $scfg = $storecfg->{ids}->{$storage_name};
6587
6588     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6589         return 1;
6590     }
6591
6592     if ($volid =~ m/\.(qcow2|qed)$/){
6593         return 1;
6594     }
6595
6596     return;
6597 }
6598
6599 sub qga_check_running {
6600     my ($vmid, $nowarn) = @_;
6601
6602     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6603     if ($@) {
6604         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6605         return 0;
6606     }
6607     return 1;
6608 }
6609
6610 sub template_create {
6611     my ($vmid, $conf, $disk) = @_;
6612
6613     my $storecfg = PVE::Storage::config();
6614
6615     PVE::QemuConfig->foreach_volume($conf, sub {
6616         my ($ds, $drive) = @_;
6617
6618         return if drive_is_cdrom($drive);
6619         return if $disk && $ds ne $disk;
6620
6621         my $volid = $drive->{file};
6622         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6623
6624         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6625         $drive->{file} = $voliddst;
6626         $conf->{$ds} = print_drive($drive);
6627         PVE::QemuConfig->write_config($vmid, $conf);
6628     });
6629 }
6630
6631 sub convert_iscsi_path {
6632     my ($path) = @_;
6633
6634     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6635         my $portal = $1;
6636         my $target = $2;
6637         my $lun = $3;
6638
6639         my $initiator_name = get_initiator_name();
6640
6641         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6642                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6643     }
6644
6645     die "cannot convert iscsi path '$path', unkown format\n";
6646 }
6647
6648 sub qemu_img_convert {
6649     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6650
6651     my $storecfg = PVE::Storage::config();
6652     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6653     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6654
6655     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6656
6657     my $cachemode;
6658     my $src_path;
6659     my $src_is_iscsi = 0;
6660     my $src_format;
6661
6662     if ($src_storeid) {
6663         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6664         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6665         $src_format = qemu_img_format($src_scfg, $src_volname);
6666         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6667         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6668         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6669     } elsif (-f $src_volid) {
6670         $src_path = $src_volid;
6671         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6672             $src_format = $1;
6673         }
6674     }
6675
6676     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6677
6678     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6679     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6680     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6681     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6682
6683     my $cmd = [];
6684     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6685     push @$cmd, '-l', "snapshot.name=$snapname"
6686         if $snapname && $src_format && $src_format eq "qcow2";
6687     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6688     push @$cmd, '-T', $cachemode if defined($cachemode);
6689
6690     if ($src_is_iscsi) {
6691         push @$cmd, '--image-opts';
6692         $src_path = convert_iscsi_path($src_path);
6693     } elsif ($src_format) {
6694         push @$cmd, '-f', $src_format;
6695     }
6696
6697     if ($dst_is_iscsi) {
6698         push @$cmd, '--target-image-opts';
6699         $dst_path = convert_iscsi_path($dst_path);
6700     } else {
6701         push @$cmd, '-O', $dst_format;
6702     }
6703
6704     push @$cmd, $src_path;
6705
6706     if (!$dst_is_iscsi && $is_zero_initialized) {
6707         push @$cmd, "zeroinit:$dst_path";
6708     } else {
6709         push @$cmd, $dst_path;
6710     }
6711
6712     my $parser = sub {
6713         my $line = shift;
6714         if($line =~ m/\((\S+)\/100\%\)/){
6715             my $percent = $1;
6716             my $transferred = int($size * $percent / 100);
6717             my $remaining = $size - $transferred;
6718
6719             print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6720         }
6721
6722     };
6723
6724     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6725     my $err = $@;
6726     die "copy failed: $err" if $err;
6727 }
6728
6729 sub qemu_img_format {
6730     my ($scfg, $volname) = @_;
6731
6732     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6733         return $1;
6734     } else {
6735         return "raw";
6736     }
6737 }
6738
6739 sub qemu_drive_mirror {
6740     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
6741
6742     $jobs = {} if !$jobs;
6743
6744     my $qemu_target;
6745     my $format;
6746     $jobs->{"drive-$drive"} = {};
6747
6748     if ($dst_volid =~ /^nbd:/) {
6749         $qemu_target = $dst_volid;
6750         $format = "nbd";
6751     } else {
6752         my $storecfg = PVE::Storage::config();
6753         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6754
6755         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6756
6757         $format = qemu_img_format($dst_scfg, $dst_volname);
6758
6759         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6760
6761         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6762     }
6763
6764     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6765     $opts->{format} = $format if $format;
6766
6767     if (defined($src_bitmap)) {
6768         $opts->{sync} = 'incremental';
6769         $opts->{bitmap} = $src_bitmap;
6770         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
6771     }
6772
6773     if (defined($bwlimit)) {
6774         $opts->{speed} = $bwlimit * 1024;
6775         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6776     } else {
6777         print "drive mirror is starting for drive-$drive\n";
6778     }
6779
6780     # if a job already runs for this device we get an error, catch it for cleanup
6781     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
6782     if (my $err = $@) {
6783         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6784         warn "$@\n" if $@;
6785         die "mirroring error: $err\n";
6786     }
6787
6788     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
6789 }
6790
6791 # $completion can be either
6792 # 'complete': wait until all jobs are ready, block-job-complete them (default)
6793 # 'cancel': wait until all jobs are ready, block-job-cancel them
6794 # 'skip': wait until all jobs are ready, return with block jobs in ready state
6795 sub qemu_drive_mirror_monitor {
6796     my ($vmid, $vmiddst, $jobs, $completion, $qga) = @_;
6797
6798     $completion //= 'complete';
6799
6800     eval {
6801         my $err_complete = 0;
6802
6803         while (1) {
6804             die "storage migration timed out\n" if $err_complete > 300;
6805
6806             my $stats = mon_cmd($vmid, "query-block-jobs");
6807
6808             my $running_mirror_jobs = {};
6809             foreach my $stat (@$stats) {
6810                 next if $stat->{type} ne 'mirror';
6811                 $running_mirror_jobs->{$stat->{device}} = $stat;
6812             }
6813
6814             my $readycounter = 0;
6815
6816             foreach my $job (keys %$jobs) {
6817
6818                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6819                     print "$job : finished\n";
6820                     delete $jobs->{$job};
6821                     next;
6822                 }
6823
6824                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6825
6826                 my $busy = $running_mirror_jobs->{$job}->{busy};
6827                 my $ready = $running_mirror_jobs->{$job}->{ready};
6828                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6829                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6830                     my $remaining = $total - $transferred;
6831                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6832
6833                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6834                 }
6835
6836                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6837             }
6838
6839             last if scalar(keys %$jobs) == 0;
6840
6841             if ($readycounter == scalar(keys %$jobs)) {
6842                 print "all mirroring jobs are ready \n";
6843                 last if $completion eq 'skip'; #do the complete later
6844
6845                 if ($vmiddst && $vmiddst != $vmid) {
6846                     my $agent_running = $qga && qga_check_running($vmid);
6847                     if ($agent_running) {
6848                         print "freeze filesystem\n";
6849                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6850                     } else {
6851                         print "suspend vm\n";
6852                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6853                     }
6854
6855                     # if we clone a disk for a new target vm, we don't switch the disk
6856                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6857
6858                     if ($agent_running) {
6859                         print "unfreeze filesystem\n";
6860                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6861                     } else {
6862                         print "resume vm\n";
6863                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6864                     }
6865
6866                     last;
6867                 } else {
6868
6869                     foreach my $job (keys %$jobs) {
6870                         # try to switch the disk if source and destination are on the same guest
6871                         print "$job: Completing block job...\n";
6872
6873                         my $op;
6874                         if ($completion eq 'complete') {
6875                             $op = 'block-job-complete';
6876                         } elsif ($completion eq 'cancel') {
6877                             $op = 'block-job-cancel';
6878                         } else {
6879                             die "invalid completion value: $completion\n";
6880                         }
6881                         eval { mon_cmd($vmid, $op, device => $job) };
6882                         if ($@ =~ m/cannot be completed/) {
6883                             print "$job: Block job cannot be completed, try again.\n";
6884                             $err_complete++;
6885                         }else {
6886                             print "$job: Completed successfully.\n";
6887                             $jobs->{$job}->{complete} = 1;
6888                         }
6889                     }
6890                 }
6891             }
6892             sleep 1;
6893         }
6894     };
6895     my $err = $@;
6896
6897     if ($err) {
6898         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6899         die "mirroring error: $err";
6900     }
6901
6902 }
6903
6904 sub qemu_blockjobs_cancel {
6905     my ($vmid, $jobs) = @_;
6906
6907     foreach my $job (keys %$jobs) {
6908         print "$job: Cancelling block job\n";
6909         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
6910         $jobs->{$job}->{cancel} = 1;
6911     }
6912
6913     while (1) {
6914         my $stats = mon_cmd($vmid, "query-block-jobs");
6915
6916         my $running_jobs = {};
6917         foreach my $stat (@$stats) {
6918             $running_jobs->{$stat->{device}} = $stat;
6919         }
6920
6921         foreach my $job (keys %$jobs) {
6922
6923             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6924                 print "$job: Done.\n";
6925                 delete $jobs->{$job};
6926             }
6927         }
6928
6929         last if scalar(keys %$jobs) == 0;
6930
6931         sleep 1;
6932     }
6933 }
6934
6935 sub clone_disk {
6936     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6937         $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
6938
6939     my $newvolid;
6940
6941     if (!$full) {
6942         print "create linked clone of drive $drivename ($drive->{file})\n";
6943         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6944         push @$newvollist, $newvolid;
6945     } else {
6946
6947         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6948         $storeid = $storage if $storage;
6949
6950         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6951
6952         print "create full clone of drive $drivename ($drive->{file})\n";
6953         my $name = undef;
6954         my $size = undef;
6955         if (drive_is_cloudinit($drive)) {
6956             $name = "vm-$newvmid-cloudinit";
6957             $name .= ".$dst_format" if $dst_format ne 'raw';
6958             $snapname = undef;
6959             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
6960         } elsif ($drivename eq 'efidisk0') {
6961             $size = get_efivars_size($conf);
6962         } else {
6963             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6964         }
6965         $size /= 1024;
6966         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, $size);
6967         push @$newvollist, $newvolid;
6968
6969         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6970
6971         if (drive_is_cloudinit($drive)) {
6972             goto no_data_clone;
6973         }
6974
6975         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6976         if (!$running || $snapname) {
6977             # TODO: handle bwlimits
6978             if ($drivename eq 'efidisk0') {
6979                 # the relevant data on the efidisk may be smaller than the source
6980                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
6981                 # that is given by the OVMF_VARS.fd
6982                 my $src_path = PVE::Storage::path($storecfg, $drive->{file});
6983                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
6984                 run_command(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=1", "count=$size",
6985                     "if=$src_path", "of=$dst_path"]);
6986             } else {
6987                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6988             }
6989         } else {
6990
6991             my $kvmver = get_running_qemu_version ($vmid);
6992             if (!min_version($kvmver, 2, 7)) {
6993                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6994                     if $drive->{iothread};
6995             }
6996
6997             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs,
6998                 $completion, $qga, $bwlimit);
6999         }
7000     }
7001
7002 no_data_clone:
7003     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
7004
7005     my $disk = $drive;
7006     $disk->{format} = undef;
7007     $disk->{file} = $newvolid;
7008     $disk->{size} = $size;
7009
7010     return $disk;
7011 }
7012
7013 sub get_running_qemu_version {
7014     my ($vmid) = @_;
7015     my $res = mon_cmd($vmid, "query-version");
7016     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7017 }
7018
7019 sub qemu_use_old_bios_files {
7020     my ($machine_type) = @_;
7021
7022     return if !$machine_type;
7023
7024     my $use_old_bios_files = undef;
7025
7026     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7027         $machine_type = $1;
7028         $use_old_bios_files = 1;
7029     } else {
7030         my $version = extract_version($machine_type, kvm_user_version());
7031         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7032         # load new efi bios files on migration. So this hack is required to allow
7033         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7034         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7035         $use_old_bios_files = !min_version($version, 2, 4);
7036     }
7037
7038     return ($use_old_bios_files, $machine_type);
7039 }
7040
7041 sub get_efivars_size {
7042     my ($conf) = @_;
7043     my $arch = get_vm_arch($conf);
7044     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7045     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
7046     return -s $ovmf_vars;
7047 }
7048
7049 sub update_efidisk_size {
7050     my ($conf) = @_;
7051
7052     return if !defined($conf->{efidisk0});
7053
7054     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
7055     $disk->{size} = get_efivars_size($conf);
7056     $conf->{efidisk0} = print_drive($disk);
7057
7058     return;
7059 }
7060
7061 sub create_efidisk($$$$$) {
7062     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7063
7064     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7065     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7066
7067     my $vars_size_b = -s $ovmf_vars;
7068     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7069     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7070     PVE::Storage::activate_volumes($storecfg, [$volid]);
7071
7072     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7073     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7074
7075     return ($volid, $size/1024);
7076 }
7077
7078 sub vm_iothreads_list {
7079     my ($vmid) = @_;
7080
7081     my $res = mon_cmd($vmid, 'query-iothreads');
7082
7083     my $iothreads = {};
7084     foreach my $iothread (@$res) {
7085         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7086     }
7087
7088     return $iothreads;
7089 }
7090
7091 sub scsihw_infos {
7092     my ($conf, $drive) = @_;
7093
7094     my $maxdev = 0;
7095
7096     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7097         $maxdev = 7;
7098     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7099         $maxdev = 1;
7100     } else {
7101         $maxdev = 256;
7102     }
7103
7104     my $controller = int($drive->{index} / $maxdev);
7105     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
7106         ? "virtioscsi"
7107         : "scsihw";
7108
7109     return ($maxdev, $controller, $controller_prefix);
7110 }
7111
7112 sub windows_version {
7113     my ($ostype) = @_;
7114
7115     return 0 if !$ostype;
7116
7117     my $winversion = 0;
7118
7119     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7120         $winversion = 5;
7121     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7122         $winversion = 6;
7123     } elsif ($ostype =~ m/^win(\d+)$/) {
7124         $winversion = $1;
7125     }
7126
7127     return $winversion;
7128 }
7129
7130 sub resolve_dst_disk_format {
7131         my ($storecfg, $storeid, $src_volname, $format) = @_;
7132         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7133
7134         if (!$format) {
7135             # if no target format is specified, use the source disk format as hint
7136             if ($src_volname) {
7137                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7138                 $format = qemu_img_format($scfg, $src_volname);
7139             } else {
7140                 return $defFormat;
7141             }
7142         }
7143
7144         # test if requested format is supported - else use default
7145         my $supported = grep { $_ eq $format } @$validFormats;
7146         $format = $defFormat if !$supported;
7147         return $format;
7148 }
7149
7150 # NOTE: if this logic changes, please update docs & possibly gui logic
7151 sub find_vmstate_storage {
7152     my ($conf, $storecfg) = @_;
7153
7154     # first, return storage from conf if set
7155     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7156
7157     my ($target, $shared, $local);
7158
7159     foreach_storage_used_by_vm($conf, sub {
7160         my ($sid) = @_;
7161         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7162         my $dst = $scfg->{shared} ? \$shared : \$local;
7163         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7164     });
7165
7166     # second, use shared storage where VM has at least one disk
7167     # third, use local storage where VM has at least one disk
7168     # fall back to local storage
7169     $target = $shared // $local // 'local';
7170
7171     return $target;
7172 }
7173
7174 sub generate_uuid {
7175     my ($uuid, $uuid_str);
7176     UUID::generate($uuid);
7177     UUID::unparse($uuid, $uuid_str);
7178     return $uuid_str;
7179 }
7180
7181 sub generate_smbios1_uuid {
7182     return "uuid=".generate_uuid();
7183 }
7184
7185 sub nbd_stop {
7186     my ($vmid) = @_;
7187
7188     mon_cmd($vmid, 'nbd-server-stop');
7189 }
7190
7191 sub create_reboot_request {
7192     my ($vmid) = @_;
7193     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7194         or die "failed to create reboot trigger file: $!\n";
7195     close($fh);
7196 }
7197
7198 sub clear_reboot_request {
7199     my ($vmid) = @_;
7200     my $path = "/run/qemu-server/$vmid.reboot";
7201     my $res = 0;
7202
7203     $res = unlink($path);
7204     die "could not remove reboot request for $vmid: $!"
7205         if !$res && $! != POSIX::ENOENT;
7206
7207     return $res;
7208 }
7209
7210 sub bootorder_from_legacy {
7211     my ($conf, $bootcfg) = @_;
7212
7213     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
7214     my $bootindex_hash = {};
7215     my $i = 1;
7216     foreach my $o (split(//, $boot)) {
7217         $bootindex_hash->{$o} = $i*100;
7218         $i++;
7219     }
7220
7221     my $bootorder = {};
7222
7223     PVE::QemuConfig->foreach_volume($conf, sub {
7224         my ($ds, $drive) = @_;
7225
7226         if (drive_is_cdrom ($drive, 1)) {
7227             if ($bootindex_hash->{d}) {
7228                 $bootorder->{$ds} = $bootindex_hash->{d};
7229                 $bootindex_hash->{d} += 1;
7230             }
7231         } elsif ($bootindex_hash->{c}) {
7232             $bootorder->{$ds} = $bootindex_hash->{c}
7233                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
7234             $bootindex_hash->{c} += 1;
7235         }
7236     });
7237
7238     if ($bootindex_hash->{n}) {
7239         for (my $i = 0; $i < $MAX_NETS; $i++) {
7240             my $netname = "net$i";
7241             next if !$conf->{$netname};
7242             $bootorder->{$netname} = $bootindex_hash->{n};
7243             $bootindex_hash->{n} += 1;
7244         }
7245     }
7246
7247     return $bootorder;
7248 }
7249
7250 # Generate default device list for 'boot: order=' property. Matches legacy
7251 # default boot order, but with explicit device names. This is important, since
7252 # the fallback for when neither 'order' nor the old format is specified relies
7253 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
7254 sub get_default_bootdevices {
7255     my ($conf) = @_;
7256
7257     my @ret = ();
7258
7259     # harddisk
7260     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
7261     push @ret, $first if $first;
7262
7263     # cdrom
7264     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
7265     push @ret, $first if $first;
7266
7267     # network
7268     for (my $i = 0; $i < $MAX_NETS; $i++) {
7269         my $netname = "net$i";
7270         next if !$conf->{$netname};
7271         push @ret, $netname;
7272         last;
7273     }
7274
7275     return \@ret;
7276 }
7277
7278 # bash completion helper
7279
7280 sub complete_backup_archives {
7281     my ($cmdname, $pname, $cvalue) = @_;
7282
7283     my $cfg = PVE::Storage::config();
7284
7285     my $storeid;
7286
7287     if ($cvalue =~ m/^([^:]+):/) {
7288         $storeid = $1;
7289     }
7290
7291     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7292
7293     my $res = [];
7294     foreach my $id (keys %$data) {
7295         foreach my $item (@{$data->{$id}}) {
7296             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
7297             push @$res, $item->{volid} if defined($item->{volid});
7298         }
7299     }
7300
7301     return $res;
7302 }
7303
7304 my $complete_vmid_full = sub {
7305     my ($running) = @_;
7306
7307     my $idlist = vmstatus();
7308
7309     my $res = [];
7310
7311     foreach my $id (keys %$idlist) {
7312         my $d = $idlist->{$id};
7313         if (defined($running)) {
7314             next if $d->{template};
7315             next if $running && $d->{status} ne 'running';
7316             next if !$running && $d->{status} eq 'running';
7317         }
7318         push @$res, $id;
7319
7320     }
7321     return $res;
7322 };
7323
7324 sub complete_vmid {
7325     return &$complete_vmid_full();
7326 }
7327
7328 sub complete_vmid_stopped {
7329     return &$complete_vmid_full(0);
7330 }
7331
7332 sub complete_vmid_running {
7333     return &$complete_vmid_full(1);
7334 }
7335
7336 sub complete_storage {
7337
7338     my $cfg = PVE::Storage::config();
7339     my $ids = $cfg->{ids};
7340
7341     my $res = [];
7342     foreach my $sid (keys %$ids) {
7343         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7344         next if !$ids->{$sid}->{content}->{images};
7345         push @$res, $sid;
7346     }
7347
7348     return $res;
7349 }
7350
7351 sub complete_migration_storage {
7352     my ($cmd, $param, $current_value, $all_args) = @_;
7353
7354     my $targetnode = @$all_args[1];
7355
7356     my $cfg = PVE::Storage::config();
7357     my $ids = $cfg->{ids};
7358
7359     my $res = [];
7360     foreach my $sid (keys %$ids) {
7361         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7362         next if !$ids->{$sid}->{content}->{images};
7363         push @$res, $sid;
7364     }
7365
7366     return $res;
7367 }
7368
7369 1;