PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::CGroup;
  31 use PVE::DataCenterConfig;
  32 use PVE::Exception qw(raise raise_param_exc);
  33 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  34 use PVE::INotify;
  35 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  36 use PVE::ProcFSTools;
  37 use PVE::PBSClient;
  38 use PVE::RPCEnvironment;
  39 use PVE::Storage;
  40 use PVE::SysFSTools;
  41 use PVE::Systemd;
  42 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  43
  44 use PVE::QMPClient;
  45 use PVE::QemuConfig;
  46 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  47 use PVE::QemuServer::Cloudinit;
  48 use PVE::QemuServer::CGroup;
  49 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  50 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom parse_drive print_drive);
  51 use PVE::QemuServer::Machine;
  52 use PVE::QemuServer::Memory;
  53 use PVE::QemuServer::Monitor qw(mon_cmd);
  54 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  55 use PVE::QemuServer::USB qw(parse_usb_device);
  56
  57 my $have_sdn;
  58 eval {
  59     require PVE::Network::SDN::Zones;
  60     $have_sdn = 1;
  61 };
  62
  63 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  64 my $OVMF = {
  65     x86_64 => [
  66         "$EDK2_FW_BASE/OVMF_CODE.fd",
  67         "$EDK2_FW_BASE/OVMF_VARS.fd"
  68     ],
  69     aarch64 => [
  70         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  71         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  72     ],
  73 };
  74
  75 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  76
  77 # Note about locking: we use flock on the config file protect
  78 # against concurent actions.
  79 # Aditionaly, we have a 'lock' setting in the config file. This
  80 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  81 # allowed when such lock is set. But you can ignore this kind of
  82 # lock with the --skiplock flag.
  83
  84 cfs_register_file('/qemu-server/',
  85                   \&parse_vm_config,
  86                   \&write_vm_config);
  87
  88 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  89     description => "Some command save/restore state from this location.",
  90     type => 'string',
  91     maxLength => 128,
  92     optional => 1,
  93 });
  94
  95 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  96         description => "Specifies the Qemu machine type.",
  97         type => 'string',
  98         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
  99         maxLength => 40,
 100         optional => 1,
 101 });
 102
 103
 104 sub map_storage {
 105     my ($map, $source) = @_;
 106
 107     return $source if !defined($map);
 108
 109     return $map->{entries}->{$source}
 110         if $map->{entries} && defined($map->{entries}->{$source});
 111
 112     return $map->{default} if $map->{default};
 113
 114     # identity (fallback)
 115     return $source;
 116 }
 117
 118 PVE::JSONSchema::register_standard_option('pve-targetstorage', {
 119     description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
 120     type => 'string',
 121     format => 'storagepair-list',
 122     optional => 1,
 123 });
 124
 125 #no warnings 'redefine';
 126
 127 my $nodename_cache;
 128 sub nodename {
 129     $nodename_cache //= PVE::INotify::nodename();
 130     return $nodename_cache;
 131 }
 132
 133 my $watchdog_fmt = {
 134     model => {
 135         default_key => 1,
 136         type => 'string',
 137         enum => [qw(i6300esb ib700)],
 138         description => "Watchdog type to emulate.",
 139         default => 'i6300esb',
 140         optional => 1,
 141     },
 142     action => {
 143         type => 'string',
 144         enum => [qw(reset shutdown poweroff pause debug none)],
 145         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 146         optional => 1,
 147     },
 148 };
 149 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 150
 151 my $agent_fmt = {
 152     enabled => {
 153         description => "Enable/disable Qemu GuestAgent.",
 154         type => 'boolean',
 155         default => 0,
 156         default_key => 1,
 157     },
 158     fstrim_cloned_disks => {
 159         description => "Run fstrim after moving a disk or migrating the VM.",
 160         type => 'boolean',
 161         optional => 1,
 162         default => 0
 163     },
 164     type => {
 165         description => "Select the agent type",
 166         type => 'string',
 167         default => 'virtio',
 168         optional => 1,
 169         enum => [qw(virtio isa)],
 170     },
 171 };
 172
 173 my $vga_fmt = {
 174     type => {
 175         description => "Select the VGA type.",
 176         type => 'string',
 177         default => 'std',
 178         optional => 1,
 179         default_key => 1,
 180         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 181     },
 182     memory => {
 183         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 184         type => 'integer',
 185         optional => 1,
 186         minimum => 4,
 187         maximum => 512,
 188     },
 189 };
 190
 191 my $ivshmem_fmt = {
 192     size => {
 193         type => 'integer',
 194         minimum => 1,
 195         description => "The size of the file in MB.",
 196     },
 197     name => {
 198         type => 'string',
 199         pattern => '[a-zA-Z0-9\-]+',
 200         optional => 1,
 201         format_description => 'string',
 202         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 203     },
 204 };
 205
 206 my $audio_fmt = {
 207     device => {
 208         type => 'string',
 209         enum => [qw(ich9-intel-hda intel-hda AC97)],
 210         description =>  "Configure an audio device."
 211     },
 212     driver =>  {
 213         type => 'string',
 214         enum => ['spice', 'none'],
 215         default => 'spice',
 216         optional => 1,
 217         description => "Driver backend for the audio device."
 218     },
 219 };
 220
 221 my $spice_enhancements_fmt = {
 222     foldersharing => {
 223         type => 'boolean',
 224         optional => 1,
 225         default => '0',
 226         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 227     },
 228     videostreaming =>  {
 229         type => 'string',
 230         enum => ['off', 'all', 'filter'],
 231         default => 'off',
 232         optional => 1,
 233         description => "Enable video streaming. Uses compression for detected video streams."
 234     },
 235 };
 236
 237 my $rng_fmt = {
 238     source => {
 239         type => 'string',
 240         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 241         default_key => 1,
 242         description => "The file on the host to gather entropy from. In most"
 243                      . " cases /dev/urandom should be preferred over /dev/random"
 244                      . " to avoid entropy-starvation issues on the host. Using"
 245                      . " urandom does *not* decrease security in any meaningful"
 246                      . " way, as it's still seeded from real entropy, and the"
 247                      . " bytes provided will most likely be mixed with real"
 248                      . " entropy on the guest as well. /dev/hwrng can be used"
 249                      . " to pass through a hardware RNG from the host.",
 250     },
 251     max_bytes => {
 252         type => 'integer',
 253         description => "Maximum bytes of entropy injected into the guest every"
 254                      . " 'period' milliseconds. Prefer a lower value when using"
 255                      . " /dev/random as source. Use 0 to disable limiting"
 256                      . " (potentially dangerous!).",
 257         optional => 1,
 258
 259         # default is 1 KiB/s, provides enough entropy to the guest to avoid
 260         # boot-starvation issues (e.g. systemd etc...) while allowing no chance
 261         # of overwhelming the host, provided we're reading from /dev/urandom
 262         default => 1024,
 263     },
 264     period => {
 265         type => 'integer',
 266         description => "Every 'period' milliseconds the entropy-injection quota"
 267                      . " is reset, allowing the guest to retrieve another"
 268                      . " 'max_bytes' of entropy.",
 269         optional => 1,
 270         default => 1000,
 271     },
 272 };
 273
 274 my $confdesc = {
 275     onboot => {
 276         optional => 1,
 277         type => 'boolean',
 278         description => "Specifies whether a VM will be started during system bootup.",
 279         default => 0,
 280     },
 281     autostart => {
 282         optional => 1,
 283         type => 'boolean',
 284         description => "Automatic restart after crash (currently ignored).",
 285         default => 0,
 286     },
 287     hotplug => {
 288         optional => 1,
 289         type => 'string', format => 'pve-hotplug-features',
 290         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 291         default => 'network,disk,usb',
 292     },
 293     reboot => {
 294         optional => 1,
 295         type => 'boolean',
 296         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 297         default => 1,
 298     },
 299     lock => {
 300         optional => 1,
 301         type => 'string',
 302         description => "Lock/unlock the VM.",
 303         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 304     },
 305     cpulimit => {
 306         optional => 1,
 307         type => 'number',
 308         description => "Limit of CPU usage.",
 309         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 310         minimum => 0,
 311         maximum => 128,
 312         default => 0,
 313     },
 314     cpuunits => {
 315         optional => 1,
 316         type => 'integer',
 317         description => "CPU weight for a VM.",
 318         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 319         minimum => 2,
 320         maximum => 262144,
 321         default => 1024,
 322     },
 323     memory => {
 324         optional => 1,
 325         type => 'integer',
 326         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 327         minimum => 16,
 328         default => 512,
 329     },
 330     balloon => {
 331         optional => 1,
 332         type => 'integer',
 333         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 334         minimum => 0,
 335     },
 336     shares => {
 337         optional => 1,
 338         type => 'integer',
 339         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 340         minimum => 0,
 341         maximum => 50000,
 342         default => 1000,
 343     },
 344     keyboard => {
 345         optional => 1,
 346         type => 'string',
 347         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 348                        "It should not be necessary to set it.",
 349         enum => PVE::Tools::kvmkeymaplist(),
 350         default => undef,
 351     },
 352     name => {
 353         optional => 1,
 354         type => 'string', format => 'dns-name',
 355         description => "Set a name for the VM. Only used on the configuration web interface.",
 356     },
 357     scsihw => {
 358         optional => 1,
 359         type => 'string',
 360         description => "SCSI controller model",
 361         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 362         default => 'lsi',
 363     },
 364     description => {
 365         optional => 1,
 366         type => 'string',
 367         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 368     },
 369     ostype => {
 370         optional => 1,
 371         type => 'string',
 372         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 373         description => "Specify guest operating system.",
 374         verbose_description => <<EODESC,
 375 Specify guest operating system. This is used to enable special
 376 optimization/features for specific operating systems:
 377
 378 [horizontal]
 379 other;; unspecified OS
 380 wxp;; Microsoft Windows XP
 381 w2k;; Microsoft Windows 2000
 382 w2k3;; Microsoft Windows 2003
 383 w2k8;; Microsoft Windows 2008
 384 wvista;; Microsoft Windows Vista
 385 win7;; Microsoft Windows 7
 386 win8;; Microsoft Windows 8/2012/2012r2
 387 win10;; Microsoft Windows 10/2016
 388 l24;; Linux 2.4 Kernel
 389 l26;; Linux 2.6 - 5.X Kernel
 390 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 391 EODESC
 392     },
 393     boot => {
 394         optional => 1,
 395         type => 'string', format => 'pve-qm-boot',
 396         description => "Specify guest boot order. Use with 'order=', usage with"
 397                      . " no key or 'legacy=' is deprecated.",
 398     },
 399     bootdisk => {
 400         optional => 1,
 401         type => 'string', format => 'pve-qm-bootdisk',
 402         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 403         pattern => '(ide|sata|scsi|virtio)\d+',
 404     },
 405     smp => {
 406         optional => 1,
 407         type => 'integer',
 408         description => "The number of CPUs. Please use option -sockets instead.",
 409         minimum => 1,
 410         default => 1,
 411     },
 412     sockets => {
 413         optional => 1,
 414         type => 'integer',
 415         description => "The number of CPU sockets.",
 416         minimum => 1,
 417         default => 1,
 418     },
 419     cores => {
 420         optional => 1,
 421         type => 'integer',
 422         description => "The number of cores per socket.",
 423         minimum => 1,
 424         default => 1,
 425     },
 426     numa => {
 427         optional => 1,
 428         type => 'boolean',
 429         description => "Enable/disable NUMA.",
 430         default => 0,
 431     },
 432     hugepages => {
 433         optional => 1,
 434         type => 'string',
 435         description => "Enable/disable hugepages memory.",
 436         enum => [qw(any 2 1024)],
 437     },
 438     keephugepages => {
 439         optional => 1,
 440         type => 'boolean',
 441         default => 0,
 442         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 443             ." after VM shutdown and can be used for subsequent starts.",
 444     },
 445     vcpus => {
 446         optional => 1,
 447         type => 'integer',
 448         description => "Number of hotplugged vcpus.",
 449         minimum => 1,
 450         default => 0,
 451     },
 452     acpi => {
 453         optional => 1,
 454         type => 'boolean',
 455         description => "Enable/disable ACPI.",
 456         default => 1,
 457     },
 458     agent => {
 459         optional => 1,
 460         description => "Enable/disable Qemu GuestAgent and its properties.",
 461         type => 'string',
 462         format => $agent_fmt,
 463     },
 464     kvm => {
 465         optional => 1,
 466         type => 'boolean',
 467         description => "Enable/disable KVM hardware virtualization.",
 468         default => 1,
 469     },
 470     tdf => {
 471         optional => 1,
 472         type => 'boolean',
 473         description => "Enable/disable time drift fix.",
 474         default => 0,
 475     },
 476     localtime => {
 477         optional => 1,
 478         type => 'boolean',
 479         description => "Set the real time clock to local time. This is enabled by default if ostype"
 480             ." indicates a Microsoft OS.",
 481     },
 482     freeze => {
 483         optional => 1,
 484         type => 'boolean',
 485         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 486     },
 487     vga => {
 488         optional => 1,
 489         type => 'string', format => $vga_fmt,
 490         description => "Configure the VGA hardware.",
 491         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 492             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 493             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 494             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 495             ." display server. For win* OS you can select how many independent displays you want,"
 496             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 497             ." using a serial device as terminal.",
 498     },
 499     watchdog => {
 500         optional => 1,
 501         type => 'string', format => 'pve-qm-watchdog',
 502         description => "Create a virtual hardware watchdog device.",
 503         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 504             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 505             ." else the watchdog will reset the guest (or execute the respective action specified)",
 506     },
 507     startdate => {
 508         optional => 1,
 509         type => 'string',
 510         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 511         description => "Set the initial date of the real time clock. Valid format for date are:"
 512             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 513         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 514         default => 'now',
 515     },
 516     startup =>  get_standard_option('pve-startup-order'),
 517     template => {
 518         optional => 1,
 519         type => 'boolean',
 520         description => "Enable/disable Template.",
 521         default => 0,
 522     },
 523     args => {
 524         optional => 1,
 525         type => 'string',
 526         description => "Arbitrary arguments passed to kvm.",
 527         verbose_description => <<EODESCR,
 528 Arbitrary arguments passed to kvm, for example:
 529
 530 args: -no-reboot -no-hpet
 531
 532 NOTE: this option is for experts only.
 533 EODESCR
 534     },
 535     tablet => {
 536         optional => 1,
 537         type => 'boolean',
 538         default => 1,
 539         description => "Enable/disable the USB tablet device.",
 540         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 541             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 542             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 543             ." may consider disabling this to save some context switches. This is turned off by"
 544             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 545     },
 546     migrate_speed => {
 547         optional => 1,
 548         type => 'integer',
 549         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 550         minimum => 0,
 551         default => 0,
 552     },
 553     migrate_downtime => {
 554         optional => 1,
 555         type => 'number',
 556         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 557         minimum => 0,
 558         default => 0.1,
 559     },
 560     cdrom => {
 561         optional => 1,
 562         type => 'string', format => 'pve-qm-ide',
 563         typetext => '<volume>',
 564         description => "This is an alias for option -ide2",
 565     },
 566     cpu => {
 567         optional => 1,
 568         description => "Emulated CPU type.",
 569         type => 'string',
 570         format => 'pve-vm-cpu-conf',
 571     },
 572     parent => get_standard_option('pve-snapshot-name', {
 573         optional => 1,
 574         description => "Parent snapshot name. This is used internally, and should not be modified.",
 575     }),
 576     snaptime => {
 577         optional => 1,
 578         description => "Timestamp for snapshots.",
 579         type => 'integer',
 580         minimum => 0,
 581     },
 582     vmstate => {
 583         optional => 1,
 584         type => 'string', format => 'pve-volume-id',
 585         description => "Reference to a volume which stores the VM state. This is used internally"
 586             ." for snapshots.",
 587     },
 588     vmstatestorage => get_standard_option('pve-storage-id', {
 589         description => "Default storage for VM state volumes/files.",
 590         optional => 1,
 591     }),
 592     runningmachine => get_standard_option('pve-qemu-machine', {
 593         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 594             ." for snapshots.",
 595     }),
 596     runningcpu => {
 597         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 598             ." internally for snapshots.",
 599         optional => 1,
 600         type => 'string',
 601         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 602         format_description => 'QEMU -cpu parameter'
 603     },
 604     machine => get_standard_option('pve-qemu-machine'),
 605     arch => {
 606         description => "Virtual processor architecture. Defaults to the host.",
 607         optional => 1,
 608         type => 'string',
 609         enum => [qw(x86_64 aarch64)],
 610     },
 611     smbios1 => {
 612         description => "Specify SMBIOS type 1 fields.",
 613         type => 'string', format => 'pve-qm-smbios1',
 614         maxLength => 512,
 615         optional => 1,
 616     },
 617     protection => {
 618         optional => 1,
 619         type => 'boolean',
 620         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 621             ." remove disk operations.",
 622         default => 0,
 623     },
 624     bios => {
 625         optional => 1,
 626         type => 'string',
 627         enum => [ qw(seabios ovmf) ],
 628         description => "Select BIOS implementation.",
 629         default => 'seabios',
 630     },
 631     vmgenid => {
 632         type => 'string',
 633         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 634         format_description => 'UUID',
 635         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 636             ." to disable explicitly.",
 637         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 638             ." value identifier to the guest OS. This allows to notify the guest operating system"
 639             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 640             ." execution or creation from a template). The guest operating system notices the"
 641             ." change, and is then able to react as appropriate by marking its copies of"
 642             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 643             ."Note that auto-creation only works when done through API/CLI create or update methods"
 644             .", but not when manually editing the config file.",
 645         default => "1 (autogenerated)",
 646         optional => 1,
 647     },
 648     hookscript => {
 649         type => 'string',
 650         format => 'pve-volume-id',
 651         optional => 1,
 652         description => "Script that will be executed during various steps in the vms lifetime.",
 653     },
 654     ivshmem => {
 655         type => 'string',
 656         format => $ivshmem_fmt,
 657         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 658             ." the host.",
 659         optional => 1,
 660     },
 661     audio0 => {
 662         type => 'string',
 663         format => $audio_fmt,
 664         description => "Configure a audio device, useful in combination with QXL/Spice.",
 665         optional => 1
 666     },
 667     spice_enhancements => {
 668         type => 'string',
 669         format => $spice_enhancements_fmt,
 670         description => "Configure additional enhancements for SPICE.",
 671         optional => 1
 672     },
 673     tags => {
 674         type => 'string', format => 'pve-tag-list',
 675         description => 'Tags of the VM. This is only meta information.',
 676         optional => 1,
 677     },
 678     rng0 => {
 679         type => 'string',
 680         format => $rng_fmt,
 681         description => "Configure a VirtIO-based Random Number Generator.",
 682         optional => 1,
 683     },
 684 };
 685
 686 my $cicustom_fmt = {
 687     meta => {
 688         type => 'string',
 689         optional => 1,
 690         description => 'Specify a custom file containing all meta data passed to the VM via"
 691             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 692         format => 'pve-volume-id',
 693         format_description => 'volume',
 694     },
 695     network => {
 696         type => 'string',
 697         optional => 1,
 698         description => 'Specify a custom file containing all network data passed to the VM via'
 699             .' cloud-init.',
 700         format => 'pve-volume-id',
 701         format_description => 'volume',
 702     },
 703     user => {
 704         type => 'string',
 705         optional => 1,
 706         description => 'Specify a custom file containing all user data passed to the VM via'
 707             .' cloud-init.',
 708         format => 'pve-volume-id',
 709         format_description => 'volume',
 710     },
 711 };
 712 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 713
 714 my $confdesc_cloudinit = {
 715     citype => {
 716         optional => 1,
 717         type => 'string',
 718         description => 'Specifies the cloud-init configuration format. The default depends on the'
 719             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 720             .' and `configdrive2` for windows.',
 721         enum => ['configdrive2', 'nocloud'],
 722     },
 723     ciuser => {
 724         optional => 1,
 725         type => 'string',
 726         description => "cloud-init: User name to change ssh keys and password for instead of the"
 727             ." image's configured default user.",
 728     },
 729     cipassword => {
 730         optional => 1,
 731         type => 'string',
 732         description => 'cloud-init: Password to assign the user. Using this is generally not'
 733             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 734             .' support hashed passwords.',
 735     },
 736     cicustom => {
 737         optional => 1,
 738         type => 'string',
 739         description => 'cloud-init: Specify custom files to replace the automatically generated'
 740             .' ones at start.',
 741         format => 'pve-qm-cicustom',
 742     },
 743     searchdomain => {
 744         optional => 1,
 745         type => 'string',
 746         description => "cloud-init: Sets DNS search domains for a container. Create will'
 747             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 748             .' are set.",
 749     },
 750     nameserver => {
 751         optional => 1,
 752         type => 'string', format => 'address-list',
 753         description => "cloud-init: Sets DNS server IP address for a container. Create will'
 754             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 755             .' are set.",
 756     },
 757     sshkeys => {
 758         optional => 1,
 759         type => 'string',
 760         format => 'urlencoded',
 761         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 762     },
 763 };
 764
 765 # what about other qemu settings ?
 766 #cpu => 'string',
 767 #machine => 'string',
 768 #fda => 'file',
 769 #fdb => 'file',
 770 #mtdblock => 'file',
 771 #sd => 'file',
 772 #pflash => 'file',
 773 #snapshot => 'bool',
 774 #bootp => 'file',
 775 ##tftp => 'dir',
 776 ##smb => 'dir',
 777 #kernel => 'file',
 778 #append => 'string',
 779 #initrd => 'file',
 780 ##soundhw => 'string',
 781
 782 while (my ($k, $v) = each %$confdesc) {
 783     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 784 }
 785
 786 my $MAX_USB_DEVICES = 5;
 787 my $MAX_NETS = 32;
 788 my $MAX_SERIAL_PORTS = 4;
 789 my $MAX_PARALLEL_PORTS = 3;
 790 my $MAX_NUMA = 8;
 791
 792 my $numa_fmt = {
 793     cpus => {
 794         type => "string",
 795         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 796         description => "CPUs accessing this NUMA node.",
 797         format_description => "id[-id];...",
 798     },
 799     memory => {
 800         type => "number",
 801         description => "Amount of memory this NUMA node provides.",
 802         optional => 1,
 803     },
 804     hostnodes => {
 805         type => "string",
 806         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 807         description => "Host NUMA nodes to use.",
 808         format_description => "id[-id];...",
 809         optional => 1,
 810     },
 811     policy => {
 812         type => 'string',
 813         enum => [qw(preferred bind interleave)],
 814         description => "NUMA allocation policy.",
 815         optional => 1,
 816     },
 817 };
 818 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 819 my $numadesc = {
 820     optional => 1,
 821     type => 'string', format => $numa_fmt,
 822     description => "NUMA topology.",
 823 };
 824 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 825
 826 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 827     $confdesc->{"numa$i"} = $numadesc;
 828 }
 829
 830 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 831                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 832                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 833 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 834
 835 my $net_fmt_bridge_descr = <<__EOD__;
 836 Bridge to attach the network device to. The Proxmox VE standard bridge
 837 is called 'vmbr0'.
 838
 839 If you do not specify a bridge, we create a kvm user (NATed) network
 840 device, which provides DHCP and DNS services. The following addresses
 841 are used:
 842
 843  10.0.2.2   Gateway
 844  10.0.2.3   DNS Server
 845  10.0.2.4   SMB Server
 846
 847 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 848 __EOD__
 849
 850 my $net_fmt = {
 851     macaddr  => get_standard_option('mac-addr', {
 852         description => "MAC address. That address must be unique withing your network. This is"
 853             ." automatically generated if not specified.",
 854     }),
 855     model => {
 856         type => 'string',
 857         description => "Network Card Model. The 'virtio' model provides the best performance with"
 858             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 859             ." best to use 'e1000'.",
 860         enum => $nic_model_list,
 861         default_key => 1,
 862     },
 863     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 864     bridge => {
 865         type => 'string',
 866         description => $net_fmt_bridge_descr,
 867         format_description => 'bridge',
 868         pattern => '[-_.\w\d]+',
 869         optional => 1,
 870     },
 871     queues => {
 872         type => 'integer',
 873         minimum => 0, maximum => 16,
 874         description => 'Number of packet queues to be used on the device.',
 875         optional => 1,
 876     },
 877     rate => {
 878         type => 'number',
 879         minimum => 0,
 880         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 881         optional => 1,
 882     },
 883     tag => {
 884         type => 'integer',
 885         minimum => 1, maximum => 4094,
 886         description => 'VLAN tag to apply to packets on this interface.',
 887         optional => 1,
 888     },
 889     trunks => {
 890         type => 'string',
 891         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 892         description => 'VLAN trunks to pass through this interface.',
 893         format_description => 'vlanid[;vlanid...]',
 894         optional => 1,
 895     },
 896     firewall => {
 897         type => 'boolean',
 898         description => 'Whether this interface should be protected by the firewall.',
 899         optional => 1,
 900     },
 901     link_down => {
 902         type => 'boolean',
 903         description => 'Whether this interface should be disconnected (like pulling the plug).',
 904         optional => 1,
 905     },
 906     mtu => {
 907         type => 'integer',
 908         minimum => 1, maximum => 65520,
 909         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 910         optional => 1,
 911     },
 912 };
 913
 914 my $netdesc = {
 915     optional => 1,
 916     type => 'string', format => $net_fmt,
 917     description => "Specify network devices.",
 918 };
 919
 920 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 921
 922 my $ipconfig_fmt = {
 923     ip => {
 924         type => 'string',
 925         format => 'pve-ipv4-config',
 926         format_description => 'IPv4Format/CIDR',
 927         description => 'IPv4 address in CIDR format.',
 928         optional => 1,
 929         default => 'dhcp',
 930     },
 931     gw => {
 932         type => 'string',
 933         format => 'ipv4',
 934         format_description => 'GatewayIPv4',
 935         description => 'Default gateway for IPv4 traffic.',
 936         optional => 1,
 937         requires => 'ip',
 938     },
 939     ip6 => {
 940         type => 'string',
 941         format => 'pve-ipv6-config',
 942         format_description => 'IPv6Format/CIDR',
 943         description => 'IPv6 address in CIDR format.',
 944         optional => 1,
 945         default => 'dhcp',
 946     },
 947     gw6 => {
 948         type => 'string',
 949         format => 'ipv6',
 950         format_description => 'GatewayIPv6',
 951         description => 'Default gateway for IPv6 traffic.',
 952         optional => 1,
 953         requires => 'ip6',
 954     },
 955 };
 956 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 957 my $ipconfigdesc = {
 958     optional => 1,
 959     type => 'string', format => 'pve-qm-ipconfig',
 960     description => <<'EODESCR',
 961 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 962
 963 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 964
 965 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
 966 gateway should be provided.
 967 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 968
 969 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
 970 dhcp on IPv4.
 971 EODESCR
 972 };
 973 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 974
 975 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 976     $confdesc->{"net$i"} = $netdesc;
 977     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 978 }
 979
 980 foreach my $key (keys %$confdesc_cloudinit) {
 981     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 982 }
 983
 984 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 985 sub verify_volume_id_or_qm_path {
 986     my ($volid, $noerr) = @_;
 987
 988     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 989         return $volid;
 990     }
 991
 992     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 993     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 994     if ($@) {
 995         return if $noerr;
 996         die $@;
 997     }
 998     return $volid;
 999 }
1000
1001 my $usb_fmt = {
1002     host => {
1003         default_key => 1,
1004         type => 'string', format => 'pve-qm-usb-device',
1005         format_description => 'HOSTUSBDEVICE|spice',
1006         description => <<EODESCR,
1007 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1008
1009  'bus-port(.port)*' (decimal numbers) or
1010  'vendor_id:product_id' (hexadeciaml numbers) or
1011  'spice'
1012
1013 You can use the 'lsusb -t' command to list existing usb devices.
1014
1015 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1016 machines - use with special care.
1017
1018 The value 'spice' can be used to add a usb redirection devices for spice.
1019 EODESCR
1020     },
1021     usb3 => {
1022         optional => 1,
1023         type => 'boolean',
1024         description => "Specifies whether if given host option is a USB3 device or port.",
1025         default => 0,
1026     },
1027 };
1028
1029 my $usbdesc = {
1030     optional => 1,
1031     type => 'string', format => $usb_fmt,
1032     description => "Configure an USB device (n is 0 to 4).",
1033 };
1034 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1035
1036 my $serialdesc = {
1037         optional => 1,
1038         type => 'string',
1039         pattern => '(/dev/.+|socket)',
1040         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1041         verbose_description =>  <<EODESCR,
1042 Create a serial device inside the VM (n is 0 to 3), and pass through a
1043 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1044 host side (use 'qm terminal' to open a terminal connection).
1045
1046 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1047 use with special care.
1048
1049 CAUTION: Experimental! User reported problems with this option.
1050 EODESCR
1051 };
1052
1053 my $paralleldesc= {
1054         optional => 1,
1055         type => 'string',
1056         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1057         description =>  "Map host parallel devices (n is 0 to 2).",
1058         verbose_description =>  <<EODESCR,
1059 Map host parallel devices (n is 0 to 2).
1060
1061 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1062 machines - use with special care.
1063
1064 CAUTION: Experimental! User reported problems with this option.
1065 EODESCR
1066 };
1067
1068 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1069     $confdesc->{"parallel$i"} = $paralleldesc;
1070 }
1071
1072 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1073     $confdesc->{"serial$i"} = $serialdesc;
1074 }
1075
1076 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1077     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1078 }
1079
1080 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1081     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1082 }
1083
1084 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1085     $confdesc->{"usb$i"} = $usbdesc;
1086 }
1087
1088 my $boot_fmt = {
1089     legacy => {
1090         optional => 1,
1091         default_key => 1,
1092         type => 'string',
1093         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1094                      . " Deprecated, use 'order=' instead.",
1095         pattern => '[acdn]{1,4}',
1096         format_description => "[acdn]{1,4}",
1097
1098         # note: this is also the fallback if boot: is not given at all
1099         default => 'cdn',
1100     },
1101     order => {
1102         optional => 1,
1103         type => 'string',
1104         format => 'pve-qm-bootdev-list',
1105         format_description => "device[;device...]",
1106         description => <<EODESC,
1107 The guest will attempt to boot from devices in the order they appear here.
1108
1109 Disks, optical drives and passed-through storage USB devices will be directly
1110 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1111 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1112
1113 Note that only devices in this list will be marked as bootable and thus loaded
1114 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1115 (e.g. software-raid), you need to specify all of them here.
1116
1117 Overrides the deprecated 'legacy=[acdn]*' value when given.
1118 EODESC
1119     },
1120 };
1121 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1122
1123 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1124 sub verify_bootdev {
1125     my ($dev, $noerr) = @_;
1126
1127     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && $dev !~ m/^efidisk/;
1128
1129     my $check = sub {
1130         my ($base) = @_;
1131         return 0 if $dev !~ m/^$base\d+$/;
1132         return 0 if !$confdesc->{$dev};
1133         return 1;
1134     };
1135
1136     return $dev if $check->("net");
1137     return $dev if $check->("usb");
1138     return $dev if $check->("hostpci");
1139
1140     return if $noerr;
1141     die "invalid boot device '$dev'\n";
1142 }
1143
1144 sub print_bootorder {
1145     my ($devs) = @_;
1146     return "" if !@$devs;
1147     my $data = { order => join(';', @$devs) };
1148     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1149 }
1150
1151 my $kvm_api_version = 0;
1152
1153 sub kvm_version {
1154     return $kvm_api_version if $kvm_api_version;
1155
1156     open my $fh, '<', '/dev/kvm' or return;
1157
1158     # 0xae00 => KVM_GET_API_VERSION
1159     $kvm_api_version = ioctl($fh, 0xae00, 0);
1160     close($fh);
1161
1162     return $kvm_api_version;
1163 }
1164
1165 my $kvm_user_version = {};
1166 my $kvm_mtime = {};
1167
1168 sub kvm_user_version {
1169     my ($binary) = @_;
1170
1171     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1172     my $st = stat($binary);
1173
1174     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1175     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1176         $cachedmtime == $st->mtime;
1177
1178     $kvm_user_version->{$binary} = 'unknown';
1179     $kvm_mtime->{$binary} = $st->mtime;
1180
1181     my $code = sub {
1182         my $line = shift;
1183         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1184             $kvm_user_version->{$binary} = $2;
1185         }
1186     };
1187
1188     eval { run_command([$binary, '--version'], outfunc => $code); };
1189     warn $@ if $@;
1190
1191     return $kvm_user_version->{$binary};
1192
1193 }
1194 my sub extract_version {
1195     my ($machine_type, $version) = @_;
1196     $version = kvm_user_version() if !defined($version);
1197     PVE::QemuServer::Machine::extract_version($machine_type, $version)
1198 }
1199
1200 sub kernel_has_vhost_net {
1201     return -c '/dev/vhost-net';
1202 }
1203
1204 sub option_exists {
1205     my $key = shift;
1206     return defined($confdesc->{$key});
1207 }
1208
1209 my $cdrom_path;
1210 sub get_cdrom_path {
1211
1212     return  $cdrom_path if $cdrom_path;
1213
1214     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1215     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1216     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1217 }
1218
1219 sub get_iso_path {
1220     my ($storecfg, $vmid, $cdrom) = @_;
1221
1222     if ($cdrom eq 'cdrom') {
1223         return get_cdrom_path();
1224     } elsif ($cdrom eq 'none') {
1225         return '';
1226     } elsif ($cdrom =~ m|^/|) {
1227         return $cdrom;
1228     } else {
1229         return PVE::Storage::path($storecfg, $cdrom);
1230     }
1231 }
1232
1233 # try to convert old style file names to volume IDs
1234 sub filename_to_volume_id {
1235     my ($vmid, $file, $media) = @_;
1236
1237      if (!($file eq 'none' || $file eq 'cdrom' ||
1238           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1239
1240         return if $file =~ m|/|;
1241
1242         if ($media && $media eq 'cdrom') {
1243             $file = "local:iso/$file";
1244         } else {
1245             $file = "local:$vmid/$file";
1246         }
1247     }
1248
1249     return $file;
1250 }
1251
1252 sub verify_media_type {
1253     my ($opt, $vtype, $media) = @_;
1254
1255     return if !$media;
1256
1257     my $etype;
1258     if ($media eq 'disk') {
1259         $etype = 'images';
1260     } elsif ($media eq 'cdrom') {
1261         $etype = 'iso';
1262     } else {
1263         die "internal error";
1264     }
1265
1266     return if ($vtype eq $etype);
1267
1268     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1269 }
1270
1271 sub cleanup_drive_path {
1272     my ($opt, $storecfg, $drive) = @_;
1273
1274     # try to convert filesystem paths to volume IDs
1275
1276     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1277         ($drive->{file} !~ m|^/dev/.+|) &&
1278         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1279         ($drive->{file} !~ m/^\d+$/)) {
1280         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1281         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1282             if !$vtype;
1283         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1284         verify_media_type($opt, $vtype, $drive->{media});
1285         $drive->{file} = $volid;
1286     }
1287
1288     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1289 }
1290
1291 sub parse_hotplug_features {
1292     my ($data) = @_;
1293
1294     my $res = {};
1295
1296     return $res if $data eq '0';
1297
1298     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1299
1300     foreach my $feature (PVE::Tools::split_list($data)) {
1301         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1302             $res->{$1} = 1;
1303         } else {
1304             die "invalid hotplug feature '$feature'\n";
1305         }
1306     }
1307     return $res;
1308 }
1309
1310 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1311 sub pve_verify_hotplug_features {
1312     my ($value, $noerr) = @_;
1313
1314     return $value if parse_hotplug_features($value);
1315
1316     return if $noerr;
1317
1318     die "unable to parse hotplug option\n";
1319 }
1320
1321 sub scsi_inquiry {
1322     my($fh, $noerr) = @_;
1323
1324     my $SG_IO = 0x2285;
1325     my $SG_GET_VERSION_NUM = 0x2282;
1326
1327     my $versionbuf = "\x00" x 8;
1328     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1329     if (!$ret) {
1330         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1331         return;
1332     }
1333     my $version = unpack("I", $versionbuf);
1334     if ($version < 30000) {
1335         die "scsi generic interface too old\n"  if !$noerr;
1336         return;
1337     }
1338
1339     my $buf = "\x00" x 36;
1340     my $sensebuf = "\x00" x 8;
1341     my $cmd = pack("C x3 C x1", 0x12, 36);
1342
1343     # see /usr/include/scsi/sg.h
1344     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1345
1346     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1347                       length($sensebuf), 0, length($buf), $buf,
1348                       $cmd, $sensebuf, 6000);
1349
1350     $ret = ioctl($fh, $SG_IO, $packet);
1351     if (!$ret) {
1352         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1353         return;
1354     }
1355
1356     my @res = unpack($sg_io_hdr_t, $packet);
1357     if ($res[17] || $res[18]) {
1358         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1359         return;
1360     }
1361
1362     my $res = {};
1363     (my $byte0, my $byte1, $res->{vendor},
1364      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1365
1366     $res->{removable} = $byte1 & 128 ? 1 : 0;
1367     $res->{type} = $byte0 & 31;
1368
1369     return $res;
1370 }
1371
1372 sub path_is_scsi {
1373     my ($path) = @_;
1374
1375     my $fh = IO::File->new("+<$path") || return;
1376     my $res = scsi_inquiry($fh, 1);
1377     close($fh);
1378
1379     return $res;
1380 }
1381
1382 sub print_tabletdevice_full {
1383     my ($conf, $arch) = @_;
1384
1385     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1386
1387     # we use uhci for old VMs because tablet driver was buggy in older qemu
1388     my $usbbus;
1389     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1390         $usbbus = 'ehci';
1391     } else {
1392         $usbbus = 'uhci';
1393     }
1394
1395     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1396 }
1397
1398 sub print_keyboarddevice_full {
1399     my ($conf, $arch, $machine) = @_;
1400
1401     return if $arch ne 'aarch64';
1402
1403     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1404 }
1405
1406 sub print_drivedevice_full {
1407     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1408
1409     my $device = '';
1410     my $maxdev = 0;
1411
1412     my $drive_id = "$drive->{interface}$drive->{index}";
1413     if ($drive->{interface} eq 'virtio') {
1414         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1415         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1416         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1417     } elsif ($drive->{interface} eq 'scsi') {
1418
1419         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1420         my $unit = $drive->{index} % $maxdev;
1421         my $devicetype = 'hd';
1422         my $path = '';
1423         if (drive_is_cdrom($drive)) {
1424             $devicetype = 'cd';
1425         } else {
1426             if ($drive->{file} =~ m|^/|) {
1427                 $path = $drive->{file};
1428                 if (my $info = path_is_scsi($path)) {
1429                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1430                         $devicetype = 'block';
1431                     } elsif ($info->{type} == 1) { # tape
1432                         $devicetype = 'generic';
1433                     }
1434                 }
1435             } else {
1436                  $path = PVE::Storage::path($storecfg, $drive->{file});
1437             }
1438
1439             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1440             my $version = extract_version($machine_type, kvm_user_version());
1441             if ($path =~ m/^iscsi\:\/\// &&
1442                !min_version($version, 4, 1)) {
1443                 $devicetype = 'generic';
1444             }
1445         }
1446
1447         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1448             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1449         } else {
1450             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1451                 .",lun=$drive->{index}";
1452         }
1453         $device .= ",drive=drive-$drive_id,id=$drive_id";
1454
1455         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1456             $device .= ",rotation_rate=1";
1457         }
1458         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1459
1460     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1461         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1462         my $controller = int($drive->{index} / $maxdev);
1463         my $unit = $drive->{index} % $maxdev;
1464         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1465
1466         $device = "ide-$devicetype";
1467         if ($drive->{interface} eq 'ide') {
1468             $device .= ",bus=ide.$controller,unit=$unit";
1469         } else {
1470             $device .= ",bus=ahci$controller.$unit";
1471         }
1472         $device .= ",drive=drive-$drive_id,id=$drive_id";
1473
1474         if ($devicetype eq 'hd') {
1475             if (my $model = $drive->{model}) {
1476                 $model = URI::Escape::uri_unescape($model);
1477                 $device .= ",model=$model";
1478             }
1479             if ($drive->{ssd}) {
1480                 $device .= ",rotation_rate=1";
1481             }
1482         }
1483         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1484     } elsif ($drive->{interface} eq 'usb') {
1485         die "implement me";
1486         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1487     } else {
1488         die "unsupported interface type";
1489     }
1490
1491     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1492
1493     if (my $serial = $drive->{serial}) {
1494         $serial = URI::Escape::uri_unescape($serial);
1495         $device .= ",serial=$serial";
1496     }
1497
1498
1499     return $device;
1500 }
1501
1502 sub get_initiator_name {
1503     my $initiator;
1504
1505     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1506     while (defined(my $line = <$fh>)) {
1507         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1508         $initiator = $1;
1509         last;
1510     }
1511     $fh->close();
1512
1513     return $initiator;
1514 }
1515
1516 sub print_drive_commandline_full {
1517     my ($storecfg, $vmid, $drive) = @_;
1518
1519     my $path;
1520     my $volid = $drive->{file};
1521     my $format;
1522
1523     if (drive_is_cdrom($drive)) {
1524         $path = get_iso_path($storecfg, $vmid, $volid);
1525     } else {
1526         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1527         if ($storeid) {
1528             $path = PVE::Storage::path($storecfg, $volid);
1529             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1530             $format = qemu_img_format($scfg, $volname);
1531         } else {
1532             $path = $volid;
1533             $format = "raw";
1534         }
1535    }
1536
1537     my $opts = '';
1538     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1539     foreach my $o (@qemu_drive_options) {
1540         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1541     }
1542
1543     # snapshot only accepts on|off
1544     if (defined($drive->{snapshot})) {
1545         my $v = $drive->{snapshot} ? 'on' : 'off';
1546         $opts .= ",snapshot=$v";
1547     }
1548
1549     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1550         my ($dir, $qmpname) = @$type;
1551         if (my $v = $drive->{"mbps$dir"}) {
1552             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1553         }
1554         if (my $v = $drive->{"mbps${dir}_max"}) {
1555             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1556         }
1557         if (my $v = $drive->{"bps${dir}_max_length"}) {
1558             $opts .= ",throttling.bps$qmpname-max-length=$v";
1559         }
1560         if (my $v = $drive->{"iops${dir}"}) {
1561             $opts .= ",throttling.iops$qmpname=$v";
1562         }
1563         if (my $v = $drive->{"iops${dir}_max"}) {
1564             $opts .= ",throttling.iops$qmpname-max=$v";
1565         }
1566         if (my $v = $drive->{"iops${dir}_max_length"}) {
1567             $opts .= ",throttling.iops$qmpname-max-length=$v";
1568         }
1569     }
1570
1571     $opts .= ",format=$format" if $format && !$drive->{format};
1572
1573     my $cache_direct = 0;
1574
1575     if (my $cache = $drive->{cache}) {
1576         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1577     } elsif (!drive_is_cdrom($drive)) {
1578         $opts .= ",cache=none";
1579         $cache_direct = 1;
1580     }
1581
1582     # aio native works only with O_DIRECT
1583     if (!$drive->{aio}) {
1584         if($cache_direct) {
1585             $opts .= ",aio=native";
1586         } else {
1587             $opts .= ",aio=threads";
1588         }
1589     }
1590
1591     if (!drive_is_cdrom($drive)) {
1592         my $detectzeroes;
1593         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1594             $detectzeroes = 'off';
1595         } elsif ($drive->{discard}) {
1596             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1597         } else {
1598             # This used to be our default with discard not being specified:
1599             $detectzeroes = 'on';
1600         }
1601         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1602     }
1603
1604     my $pathinfo = $path ? "file=$path," : '';
1605
1606     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1607 }
1608
1609 sub print_netdevice_full {
1610     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1611
1612     my $device = $net->{model};
1613     if ($net->{model} eq 'virtio') {
1614          $device = 'virtio-net-pci';
1615      };
1616
1617     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1618     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1619     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1620         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1621         # and out of each queue plus one config interrupt and control vector queue
1622         my $vectors = $net->{queues} * 2 + 2;
1623         $tmpstr .= ",vectors=$vectors,mq=on";
1624     }
1625     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1626
1627     if (my $mtu = $net->{mtu}) {
1628         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1629             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1630             if ($mtu == 1) {
1631                  $mtu = $bridge_mtu;
1632             } elsif ($mtu < 576) {
1633                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1634             } elsif ($mtu > $bridge_mtu) {
1635                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1636             }
1637             $tmpstr .= ",host_mtu=$mtu";
1638         } else {
1639             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1640         }
1641     }
1642
1643     if ($use_old_bios_files) {
1644         my $romfile;
1645         if ($device eq 'virtio-net-pci') {
1646             $romfile = 'pxe-virtio.rom';
1647         } elsif ($device eq 'e1000') {
1648             $romfile = 'pxe-e1000.rom';
1649         } elsif ($device eq 'ne2k') {
1650             $romfile = 'pxe-ne2k_pci.rom';
1651         } elsif ($device eq 'pcnet') {
1652             $romfile = 'pxe-pcnet.rom';
1653         } elsif ($device eq 'rtl8139') {
1654             $romfile = 'pxe-rtl8139.rom';
1655         }
1656         $tmpstr .= ",romfile=$romfile" if $romfile;
1657     }
1658
1659     return $tmpstr;
1660 }
1661
1662 sub print_netdev_full {
1663     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1664
1665     my $i = '';
1666     if ($netid =~ m/^net(\d+)$/) {
1667         $i = int($1);
1668     }
1669
1670     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1671
1672     my $ifname = "tap${vmid}i$i";
1673
1674     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1675     die "interface name '$ifname' is too long (max 15 character)\n"
1676         if length($ifname) >= 16;
1677
1678     my $vhostparam = '';
1679     if (is_native($arch)) {
1680         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1681     }
1682
1683     my $vmname = $conf->{name} || "vm$vmid";
1684
1685     my $netdev = "";
1686     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1687
1688     if ($net->{bridge}) {
1689         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1690             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1691     } else {
1692         $netdev = "type=user,id=$netid,hostname=$vmname";
1693     }
1694
1695     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1696
1697     return $netdev;
1698 }
1699
1700 my $vga_map = {
1701     'cirrus' => 'cirrus-vga',
1702     'std' => 'VGA',
1703     'vmware' => 'vmware-svga',
1704     'virtio' => 'virtio-vga',
1705 };
1706
1707 sub print_vga_device {
1708     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1709
1710     my $type = $vga_map->{$vga->{type}};
1711     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1712         $type = 'virtio-gpu';
1713     }
1714     my $vgamem_mb = $vga->{memory};
1715
1716     my $max_outputs = '';
1717     if ($qxlnum) {
1718         $type = $id ? 'qxl' : 'qxl-vga';
1719
1720         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1721             # set max outputs so linux can have up to 4 qxl displays with one device
1722             if (min_version($machine_version, 4, 1)) {
1723                 $max_outputs = ",max_outputs=4";
1724             }
1725         }
1726     }
1727
1728     die "no devicetype for $vga->{type}\n" if !$type;
1729
1730     my $memory = "";
1731     if ($vgamem_mb) {
1732         if ($vga->{type} eq 'virtio') {
1733             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1734             $memory = ",max_hostmem=$bytes";
1735         } elsif ($qxlnum) {
1736             # from https://www.spice-space.org/multiple-monitors.html
1737             $memory = ",vgamem_mb=$vga->{memory}";
1738             my $ram = $vgamem_mb * 4;
1739             my $vram = $vgamem_mb * 2;
1740             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1741         } else {
1742             $memory = ",vgamem_mb=$vga->{memory}";
1743         }
1744     } elsif ($qxlnum && $id) {
1745         $memory = ",ram_size=67108864,vram_size=33554432";
1746     }
1747
1748     my $edidoff = "";
1749     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1750         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1751     }
1752
1753     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1754     my $vgaid = "vga" . ($id // '');
1755     my $pciaddr;
1756
1757     if ($q35 && $vgaid eq 'vga') {
1758         # the first display uses pcie.0 bus on q35 machines
1759         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1760     } else {
1761         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1762     }
1763
1764     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1765 }
1766
1767 sub parse_number_sets {
1768     my ($set) = @_;
1769     my $res = [];
1770     foreach my $part (split(/;/, $set)) {
1771         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1772             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1773             push @$res, [ $1, $2 ];
1774         } else {
1775             die "invalid range: $part\n";
1776         }
1777     }
1778     return $res;
1779 }
1780
1781 sub parse_numa {
1782     my ($data) = @_;
1783
1784     my $res = parse_property_string($numa_fmt, $data);
1785     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1786     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1787     return $res;
1788 }
1789
1790 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1791 sub parse_net {
1792     my ($data) = @_;
1793
1794     my $res = eval { parse_property_string($net_fmt, $data) };
1795     if ($@) {
1796         warn $@;
1797         return;
1798     }
1799     if (!defined($res->{macaddr})) {
1800         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1801         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1802     }
1803     return $res;
1804 }
1805
1806 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1807 sub parse_ipconfig {
1808     my ($data) = @_;
1809
1810     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1811     if ($@) {
1812         warn $@;
1813         return;
1814     }
1815
1816     if ($res->{gw} && !$res->{ip}) {
1817         warn 'gateway specified without specifying an IP address';
1818         return;
1819     }
1820     if ($res->{gw6} && !$res->{ip6}) {
1821         warn 'IPv6 gateway specified without specifying an IPv6 address';
1822         return;
1823     }
1824     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1825         warn 'gateway specified together with DHCP';
1826         return;
1827     }
1828     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1829         # gw6 + auto/dhcp
1830         warn "IPv6 gateway specified together with $res->{ip6} address";
1831         return;
1832     }
1833
1834     if (!$res->{ip} && !$res->{ip6}) {
1835         return { ip => 'dhcp', ip6 => 'dhcp' };
1836     }
1837
1838     return $res;
1839 }
1840
1841 sub print_net {
1842     my $net = shift;
1843
1844     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1845 }
1846
1847 sub add_random_macs {
1848     my ($settings) = @_;
1849
1850     foreach my $opt (keys %$settings) {
1851         next if $opt !~ m/^net(\d+)$/;
1852         my $net = parse_net($settings->{$opt});
1853         next if !$net;
1854         $settings->{$opt} = print_net($net);
1855     }
1856 }
1857
1858 sub vm_is_volid_owner {
1859     my ($storecfg, $vmid, $volid) = @_;
1860
1861     if ($volid !~  m|^/|) {
1862         my ($path, $owner);
1863         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1864         if ($owner && ($owner == $vmid)) {
1865             return 1;
1866         }
1867     }
1868
1869     return;
1870 }
1871
1872 sub vmconfig_register_unused_drive {
1873     my ($storecfg, $vmid, $conf, $drive) = @_;
1874
1875     if (drive_is_cloudinit($drive)) {
1876         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1877         warn $@ if $@;
1878     } elsif (!drive_is_cdrom($drive)) {
1879         my $volid = $drive->{file};
1880         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1881             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1882         }
1883     }
1884 }
1885
1886 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1887 my $smbios1_fmt = {
1888     uuid => {
1889         type => 'string',
1890         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1891         format_description => 'UUID',
1892         description => "Set SMBIOS1 UUID.",
1893         optional => 1,
1894     },
1895     version => {
1896         type => 'string',
1897         pattern => '[A-Za-z0-9+\/]+={0,2}',
1898         format_description => 'Base64 encoded string',
1899         description => "Set SMBIOS1 version.",
1900         optional => 1,
1901     },
1902     serial => {
1903         type => 'string',
1904         pattern => '[A-Za-z0-9+\/]+={0,2}',
1905         format_description => 'Base64 encoded string',
1906         description => "Set SMBIOS1 serial number.",
1907         optional => 1,
1908     },
1909     manufacturer => {
1910         type => 'string',
1911         pattern => '[A-Za-z0-9+\/]+={0,2}',
1912         format_description => 'Base64 encoded string',
1913         description => "Set SMBIOS1 manufacturer.",
1914         optional => 1,
1915     },
1916     product => {
1917         type => 'string',
1918         pattern => '[A-Za-z0-9+\/]+={0,2}',
1919         format_description => 'Base64 encoded string',
1920         description => "Set SMBIOS1 product ID.",
1921         optional => 1,
1922     },
1923     sku => {
1924         type => 'string',
1925         pattern => '[A-Za-z0-9+\/]+={0,2}',
1926         format_description => 'Base64 encoded string',
1927         description => "Set SMBIOS1 SKU string.",
1928         optional => 1,
1929     },
1930     family => {
1931         type => 'string',
1932         pattern => '[A-Za-z0-9+\/]+={0,2}',
1933         format_description => 'Base64 encoded string',
1934         description => "Set SMBIOS1 family string.",
1935         optional => 1,
1936     },
1937     base64 => {
1938         type => 'boolean',
1939         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
1940         optional => 1,
1941     },
1942 };
1943
1944 sub parse_smbios1 {
1945     my ($data) = @_;
1946
1947     my $res = eval { parse_property_string($smbios1_fmt, $data) };
1948     warn $@ if $@;
1949     return $res;
1950 }
1951
1952 sub print_smbios1 {
1953     my ($smbios1) = @_;
1954     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1955 }
1956
1957 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1958
1959 sub parse_watchdog {
1960     my ($value) = @_;
1961
1962     return if !$value;
1963
1964     my $res = eval { parse_property_string($watchdog_fmt, $value) };
1965     warn $@ if $@;
1966     return $res;
1967 }
1968
1969 sub parse_guest_agent {
1970     my ($conf) = @_;
1971
1972     return {} if !defined($conf->{agent});
1973
1974     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
1975     warn $@ if $@;
1976
1977     # if the agent is disabled ignore the other potentially set properties
1978     return {} if !$res->{enabled};
1979     return $res;
1980 }
1981
1982 sub get_qga_key {
1983     my ($conf, $key) = @_;
1984     return undef if !defined($conf->{agent});
1985
1986     my $agent = parse_guest_agent($conf);
1987     return $agent->{$key};
1988 }
1989
1990 sub parse_vga {
1991     my ($value) = @_;
1992
1993     return {} if !$value;
1994     my $res = eval { parse_property_string($vga_fmt, $value) };
1995     warn $@ if $@;
1996     return $res;
1997 }
1998
1999 sub parse_rng {
2000     my ($value) = @_;
2001
2002     return if !$value;
2003
2004     my $res = eval { parse_property_string($rng_fmt, $value) };
2005     warn $@ if $@;
2006     return $res;
2007 }
2008
2009 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2010 sub verify_usb_device {
2011     my ($value, $noerr) = @_;
2012
2013     return $value if parse_usb_device($value);
2014
2015     return if $noerr;
2016
2017     die "unable to parse usb device\n";
2018 }
2019
2020 # add JSON properties for create and set function
2021 sub json_config_properties {
2022     my $prop = shift;
2023
2024     foreach my $opt (keys %$confdesc) {
2025         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' ||
2026             $opt eq 'runningmachine' || $opt eq 'runningcpu';
2027         $prop->{$opt} = $confdesc->{$opt};
2028     }
2029
2030     return $prop;
2031 }
2032
2033 # return copy of $confdesc_cloudinit to generate documentation
2034 sub cloudinit_config_properties {
2035
2036     return dclone($confdesc_cloudinit);
2037 }
2038
2039 sub check_type {
2040     my ($key, $value) = @_;
2041
2042     die "unknown setting '$key'\n" if !$confdesc->{$key};
2043
2044     my $type = $confdesc->{$key}->{type};
2045
2046     if (!defined($value)) {
2047         die "got undefined value\n";
2048     }
2049
2050     if ($value =~ m/[\n\r]/) {
2051         die "property contains a line feed\n";
2052     }
2053
2054     if ($type eq 'boolean') {
2055         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2056         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2057         die "type check ('boolean') failed - got '$value'\n";
2058     } elsif ($type eq 'integer') {
2059         return int($1) if $value =~ m/^(\d+)$/;
2060         die "type check ('integer') failed - got '$value'\n";
2061     } elsif ($type eq 'number') {
2062         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2063         die "type check ('number') failed - got '$value'\n";
2064     } elsif ($type eq 'string') {
2065         if (my $fmt = $confdesc->{$key}->{format}) {
2066             PVE::JSONSchema::check_format($fmt, $value);
2067             return $value;
2068         }
2069         $value =~ s/^\"(.*)\"$/$1/;
2070         return $value;
2071     } else {
2072         die "internal error"
2073     }
2074 }
2075
2076 sub destroy_vm {
2077     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2078
2079     my $conf = PVE::QemuConfig->load_config($vmid);
2080
2081     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2082
2083     if ($conf->{template}) {
2084         # check if any base image is still used by a linked clone
2085         PVE::QemuConfig->foreach_volume($conf, sub {
2086                 my ($ds, $drive) = @_;
2087                 return if drive_is_cdrom($drive);
2088
2089                 my $volid = $drive->{file};
2090                 return if !$volid || $volid =~ m|^/|;
2091
2092                 die "base volume '$volid' is still in use by linked cloned\n"
2093                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2094
2095         });
2096     }
2097
2098     # only remove disks owned by this VM (referenced in the config)
2099     PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2100         my ($ds, $drive) = @_;
2101         return if drive_is_cdrom($drive, 1);
2102
2103         my $volid = $drive->{file};
2104         return if !$volid || $volid =~ m|^/|;
2105
2106         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2107         return if !$path || !$owner || ($owner != $vmid);
2108
2109         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2110         warn "Could not remove disk '$volid', check manually: $@" if $@;
2111     });
2112
2113     if ($purge_unreferenced) { # also remove unreferenced disk
2114         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2115         PVE::Storage::foreach_volid($vmdisks, sub {
2116             my ($volid, $sid, $volname, $d) = @_;
2117             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2118             warn $@ if $@;
2119         });
2120     }
2121
2122     if (defined $replacement_conf) {
2123         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2124     } else {
2125         PVE::QemuConfig->destroy_config($vmid);
2126     }
2127 }
2128
2129 sub parse_vm_config {
2130     my ($filename, $raw) = @_;
2131
2132     return if !defined($raw);
2133
2134     my $res = {
2135         digest => Digest::SHA::sha1_hex($raw),
2136         snapshots => {},
2137         pending => {},
2138     };
2139
2140     $filename =~ m|/qemu-server/(\d+)\.conf$|
2141         || die "got strange filename '$filename'";
2142
2143     my $vmid = $1;
2144
2145     my $conf = $res;
2146     my $descr;
2147     my $section = '';
2148
2149     my @lines = split(/\n/, $raw);
2150     foreach my $line (@lines) {
2151         next if $line =~ m/^\s*$/;
2152
2153         if ($line =~ m/^\[PENDING\]\s*$/i) {
2154             $section = 'pending';
2155             if (defined($descr)) {
2156                 $descr =~ s/\s+$//;
2157                 $conf->{description} = $descr;
2158             }
2159             $descr = undef;
2160             $conf = $res->{$section} = {};
2161             next;
2162
2163         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2164             $section = $1;
2165             if (defined($descr)) {
2166                 $descr =~ s/\s+$//;
2167                 $conf->{description} = $descr;
2168             }
2169             $descr = undef;
2170             $conf = $res->{snapshots}->{$section} = {};
2171             next;
2172         }
2173
2174         if ($line =~ m/^\#(.*)\s*$/) {
2175             $descr = '' if !defined($descr);
2176             $descr .= PVE::Tools::decode_text($1) . "\n";
2177             next;
2178         }
2179
2180         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2181             $descr = '' if !defined($descr);
2182             $descr .= PVE::Tools::decode_text($2);
2183         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2184             $conf->{snapstate} = $1;
2185         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2186             my $key = $1;
2187             my $value = $2;
2188             $conf->{$key} = $value;
2189         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2190             my $value = $1;
2191             if ($section eq 'pending') {
2192                 $conf->{delete} = $value; # we parse this later
2193             } else {
2194                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2195             }
2196         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2197             my $key = $1;
2198             my $value = $2;
2199             eval { $value = check_type($key, $value); };
2200             if ($@) {
2201                 warn "vm $vmid - unable to parse value of '$key' - $@";
2202             } else {
2203                 $key = 'ide2' if $key eq 'cdrom';
2204                 my $fmt = $confdesc->{$key}->{format};
2205                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2206                     my $v = parse_drive($key, $value);
2207                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2208                         $v->{file} = $volid;
2209                         $value = print_drive($v);
2210                     } else {
2211                         warn "vm $vmid - unable to parse value of '$key'\n";
2212                         next;
2213                     }
2214                 }
2215
2216                 $conf->{$key} = $value;
2217             }
2218         }
2219     }
2220
2221     if (defined($descr)) {
2222         $descr =~ s/\s+$//;
2223         $conf->{description} = $descr;
2224     }
2225     delete $res->{snapstate}; # just to be sure
2226
2227     return $res;
2228 }
2229
2230 sub write_vm_config {
2231     my ($filename, $conf) = @_;
2232
2233     delete $conf->{snapstate}; # just to be sure
2234
2235     if ($conf->{cdrom}) {
2236         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2237         $conf->{ide2} = $conf->{cdrom};
2238         delete $conf->{cdrom};
2239     }
2240
2241     # we do not use 'smp' any longer
2242     if ($conf->{sockets}) {
2243         delete $conf->{smp};
2244     } elsif ($conf->{smp}) {
2245         $conf->{sockets} = $conf->{smp};
2246         delete $conf->{cores};
2247         delete $conf->{smp};
2248     }
2249
2250     my $used_volids = {};
2251
2252     my $cleanup_config = sub {
2253         my ($cref, $pending, $snapname) = @_;
2254
2255         foreach my $key (keys %$cref) {
2256             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2257                 $key eq 'snapstate' || $key eq 'pending';
2258             my $value = $cref->{$key};
2259             if ($key eq 'delete') {
2260                 die "propertry 'delete' is only allowed in [PENDING]\n"
2261                     if !$pending;
2262                 # fixme: check syntax?
2263                 next;
2264             }
2265             eval { $value = check_type($key, $value); };
2266             die "unable to parse value of '$key' - $@" if $@;
2267
2268             $cref->{$key} = $value;
2269
2270             if (!$snapname && is_valid_drivename($key)) {
2271                 my $drive = parse_drive($key, $value);
2272                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2273             }
2274         }
2275     };
2276
2277     &$cleanup_config($conf);
2278
2279     &$cleanup_config($conf->{pending}, 1);
2280
2281     foreach my $snapname (keys %{$conf->{snapshots}}) {
2282         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2283         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2284     }
2285
2286     # remove 'unusedX' settings if we re-add a volume
2287     foreach my $key (keys %$conf) {
2288         my $value = $conf->{$key};
2289         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2290             delete $conf->{$key};
2291         }
2292     }
2293
2294     my $generate_raw_config = sub {
2295         my ($conf, $pending) = @_;
2296
2297         my $raw = '';
2298
2299         # add description as comment to top of file
2300         if (defined(my $descr = $conf->{description})) {
2301             if ($descr) {
2302                 foreach my $cl (split(/\n/, $descr)) {
2303                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2304                 }
2305             } else {
2306                 $raw .= "#\n" if $pending;
2307             }
2308         }
2309
2310         foreach my $key (sort keys %$conf) {
2311             next if $key =~ /^(digest|description|pending|snapshots)$/;
2312             $raw .= "$key: $conf->{$key}\n";
2313         }
2314         return $raw;
2315     };
2316
2317     my $raw = &$generate_raw_config($conf);
2318
2319     if (scalar(keys %{$conf->{pending}})){
2320         $raw .= "\n[PENDING]\n";
2321         $raw .= &$generate_raw_config($conf->{pending}, 1);
2322     }
2323
2324     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2325         $raw .= "\n[$snapname]\n";
2326         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2327     }
2328
2329     return $raw;
2330 }
2331
2332 sub load_defaults {
2333
2334     my $res = {};
2335
2336     # we use static defaults from our JSON schema configuration
2337     foreach my $key (keys %$confdesc) {
2338         if (defined(my $default = $confdesc->{$key}->{default})) {
2339             $res->{$key} = $default;
2340         }
2341     }
2342
2343     return $res;
2344 }
2345
2346 sub config_list {
2347     my $vmlist = PVE::Cluster::get_vmlist();
2348     my $res = {};
2349     return $res if !$vmlist || !$vmlist->{ids};
2350     my $ids = $vmlist->{ids};
2351     my $nodename = nodename();
2352
2353     foreach my $vmid (keys %$ids) {
2354         my $d = $ids->{$vmid};
2355         next if !$d->{node} || $d->{node} ne $nodename;
2356         next if !$d->{type} || $d->{type} ne 'qemu';
2357         $res->{$vmid}->{exists} = 1;
2358     }
2359     return $res;
2360 }
2361
2362 # test if VM uses local resources (to prevent migration)
2363 sub check_local_resources {
2364     my ($conf, $noerr) = @_;
2365
2366     my @loc_res = ();
2367
2368     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2369     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2370
2371     push @loc_res, "ivshmem" if $conf->{ivshmem};
2372
2373     foreach my $k (keys %$conf) {
2374         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2375         # sockets are safe: they will recreated be on the target side post-migrate
2376         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2377         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2378     }
2379
2380     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2381
2382     return \@loc_res;
2383 }
2384
2385 # check if used storages are available on all nodes (use by migrate)
2386 sub check_storage_availability {
2387     my ($storecfg, $conf, $node) = @_;
2388
2389     PVE::QemuConfig->foreach_volume($conf, sub {
2390         my ($ds, $drive) = @_;
2391
2392         my $volid = $drive->{file};
2393         return if !$volid;
2394
2395         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2396         return if !$sid;
2397
2398         # check if storage is available on both nodes
2399         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2400         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2401    });
2402 }
2403
2404 # list nodes where all VM images are available (used by has_feature API)
2405 sub shared_nodes {
2406     my ($conf, $storecfg) = @_;
2407
2408     my $nodelist = PVE::Cluster::get_nodelist();
2409     my $nodehash = { map { $_ => 1 } @$nodelist };
2410     my $nodename = nodename();
2411
2412     PVE::QemuConfig->foreach_volume($conf, sub {
2413         my ($ds, $drive) = @_;
2414
2415         my $volid = $drive->{file};
2416         return if !$volid;
2417
2418         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2419         if ($storeid) {
2420             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2421             if ($scfg->{disable}) {
2422                 $nodehash = {};
2423             } elsif (my $avail = $scfg->{nodes}) {
2424                 foreach my $node (keys %$nodehash) {
2425                     delete $nodehash->{$node} if !$avail->{$node};
2426                 }
2427             } elsif (!$scfg->{shared}) {
2428                 foreach my $node (keys %$nodehash) {
2429                     delete $nodehash->{$node} if $node ne $nodename
2430                 }
2431             }
2432         }
2433     });
2434
2435     return $nodehash
2436 }
2437
2438 sub check_local_storage_availability {
2439     my ($conf, $storecfg) = @_;
2440
2441     my $nodelist = PVE::Cluster::get_nodelist();
2442     my $nodehash = { map { $_ => {} } @$nodelist };
2443
2444     PVE::QemuConfig->foreach_volume($conf, sub {
2445         my ($ds, $drive) = @_;
2446
2447         my $volid = $drive->{file};
2448         return if !$volid;
2449
2450         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2451         if ($storeid) {
2452             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2453
2454             if ($scfg->{disable}) {
2455                 foreach my $node (keys %$nodehash) {
2456                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2457                 }
2458             } elsif (my $avail = $scfg->{nodes}) {
2459                 foreach my $node (keys %$nodehash) {
2460                     if (!$avail->{$node}) {
2461                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2462                     }
2463                 }
2464             }
2465         }
2466     });
2467
2468     foreach my $node (values %$nodehash) {
2469         if (my $unavail = $node->{unavailable_storages}) {
2470             $node->{unavailable_storages} = [ sort keys %$unavail ];
2471         }
2472     }
2473
2474     return $nodehash
2475 }
2476
2477 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2478 sub check_running {
2479     my ($vmid, $nocheck, $node) = @_;
2480
2481     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2482     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2483 }
2484
2485 sub vzlist {
2486
2487     my $vzlist = config_list();
2488
2489     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2490
2491     while (defined(my $de = $fd->read)) {
2492         next if $de !~ m/^(\d+)\.pid$/;
2493         my $vmid = $1;
2494         next if !defined($vzlist->{$vmid});
2495         if (my $pid = check_running($vmid)) {
2496             $vzlist->{$vmid}->{pid} = $pid;
2497         }
2498     }
2499
2500     return $vzlist;
2501 }
2502
2503 our $vmstatus_return_properties = {
2504     vmid => get_standard_option('pve-vmid'),
2505     status => {
2506         description => "Qemu process status.",
2507         type => 'string',
2508         enum => ['stopped', 'running'],
2509     },
2510     maxmem => {
2511         description => "Maximum memory in bytes.",
2512         type => 'integer',
2513         optional => 1,
2514         renderer => 'bytes',
2515     },
2516     maxdisk => {
2517         description => "Root disk size in bytes.",
2518         type => 'integer',
2519         optional => 1,
2520         renderer => 'bytes',
2521     },
2522     name => {
2523         description => "VM name.",
2524         type => 'string',
2525         optional => 1,
2526     },
2527     qmpstatus => {
2528         description => "Qemu QMP agent status.",
2529         type => 'string',
2530         optional => 1,
2531     },
2532     pid => {
2533         description => "PID of running qemu process.",
2534         type => 'integer',
2535         optional => 1,
2536     },
2537     uptime => {
2538         description => "Uptime.",
2539         type => 'integer',
2540         optional => 1,
2541         renderer => 'duration',
2542     },
2543     cpus => {
2544         description => "Maximum usable CPUs.",
2545         type => 'number',
2546         optional => 1,
2547     },
2548     lock => {
2549         description => "The current config lock, if any.",
2550         type => 'string',
2551         optional => 1,
2552     },
2553     tags => {
2554         description  => "The current configured tags, if any",
2555         type => 'string',
2556         optional => 1,
2557     },
2558 };
2559
2560 my $last_proc_pid_stat;
2561
2562 # get VM status information
2563 # This must be fast and should not block ($full == false)
2564 # We only query KVM using QMP if $full == true (this can be slow)
2565 sub vmstatus {
2566     my ($opt_vmid, $full) = @_;
2567
2568     my $res = {};
2569
2570     my $storecfg = PVE::Storage::config();
2571
2572     my $list = vzlist();
2573     my $defaults = load_defaults();
2574
2575     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2576
2577     my $cpucount = $cpuinfo->{cpus} || 1;
2578
2579     foreach my $vmid (keys %$list) {
2580         next if $opt_vmid && ($vmid ne $opt_vmid);
2581
2582         my $conf = PVE::QemuConfig->load_config($vmid);
2583
2584         my $d = { vmid => $vmid };
2585         $d->{pid} = $list->{$vmid}->{pid};
2586
2587         # fixme: better status?
2588         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2589
2590         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2591         if (defined($size)) {
2592             $d->{disk} = 0; # no info available
2593             $d->{maxdisk} = $size;
2594         } else {
2595             $d->{disk} = 0;
2596             $d->{maxdisk} = 0;
2597         }
2598
2599         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2600             * ($conf->{cores} || $defaults->{cores});
2601         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2602         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2603
2604         $d->{name} = $conf->{name} || "VM $vmid";
2605         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2606             : $defaults->{memory}*(1024*1024);
2607
2608         if ($conf->{balloon}) {
2609             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2610             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2611                 : $defaults->{shares};
2612         }
2613
2614         $d->{uptime} = 0;
2615         $d->{cpu} = 0;
2616         $d->{mem} = 0;
2617
2618         $d->{netout} = 0;
2619         $d->{netin} = 0;
2620
2621         $d->{diskread} = 0;
2622         $d->{diskwrite} = 0;
2623
2624         $d->{template} = PVE::QemuConfig->is_template($conf);
2625
2626         $d->{serial} = 1 if conf_has_serial($conf);
2627         $d->{lock} = $conf->{lock} if $conf->{lock};
2628         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2629
2630         $res->{$vmid} = $d;
2631     }
2632
2633     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2634     foreach my $dev (keys %$netdev) {
2635         next if $dev !~ m/^tap([1-9]\d*)i/;
2636         my $vmid = $1;
2637         my $d = $res->{$vmid};
2638         next if !$d;
2639
2640         $d->{netout} += $netdev->{$dev}->{receive};
2641         $d->{netin} += $netdev->{$dev}->{transmit};
2642
2643         if ($full) {
2644             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2645             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2646         }
2647
2648     }
2649
2650     my $ctime = gettimeofday;
2651
2652     foreach my $vmid (keys %$list) {
2653
2654         my $d = $res->{$vmid};
2655         my $pid = $d->{pid};
2656         next if !$pid;
2657
2658         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2659         next if !$pstat; # not running
2660
2661         my $used = $pstat->{utime} + $pstat->{stime};
2662
2663         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2664
2665         if ($pstat->{vsize}) {
2666             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2667         }
2668
2669         my $old = $last_proc_pid_stat->{$pid};
2670         if (!$old) {
2671             $last_proc_pid_stat->{$pid} = {
2672                 time => $ctime,
2673                 used => $used,
2674                 cpu => 0,
2675             };
2676             next;
2677         }
2678
2679         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2680
2681         if ($dtime > 1000) {
2682             my $dutime = $used -  $old->{used};
2683
2684             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2685             $last_proc_pid_stat->{$pid} = {
2686                 time => $ctime,
2687                 used => $used,
2688                 cpu => $d->{cpu},
2689             };
2690         } else {
2691             $d->{cpu} = $old->{cpu};
2692         }
2693     }
2694
2695     return $res if !$full;
2696
2697     my $qmpclient = PVE::QMPClient->new();
2698
2699     my $ballooncb = sub {
2700         my ($vmid, $resp) = @_;
2701
2702         my $info = $resp->{'return'};
2703         return if !$info->{max_mem};
2704
2705         my $d = $res->{$vmid};
2706
2707         # use memory assigned to VM
2708         $d->{maxmem} = $info->{max_mem};
2709         $d->{balloon} = $info->{actual};
2710
2711         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2712             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2713             $d->{freemem} = $info->{free_mem};
2714         }
2715
2716         $d->{ballooninfo} = $info;
2717     };
2718
2719     my $blockstatscb = sub {
2720         my ($vmid, $resp) = @_;
2721         my $data = $resp->{'return'} || [];
2722         my $totalrdbytes = 0;
2723         my $totalwrbytes = 0;
2724
2725         for my $blockstat (@$data) {
2726             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2727             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2728
2729             $blockstat->{device} =~ s/drive-//;
2730             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2731         }
2732         $res->{$vmid}->{diskread} = $totalrdbytes;
2733         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2734     };
2735
2736     my $statuscb = sub {
2737         my ($vmid, $resp) = @_;
2738
2739         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2740         # this fails if ballon driver is not loaded, so this must be
2741         # the last commnand (following command are aborted if this fails).
2742         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2743
2744         my $status = 'unknown';
2745         if (!defined($status = $resp->{'return'}->{status})) {
2746             warn "unable to get VM status\n";
2747             return;
2748         }
2749
2750         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2751     };
2752
2753     foreach my $vmid (keys %$list) {
2754         next if $opt_vmid && ($vmid ne $opt_vmid);
2755         next if !$res->{$vmid}->{pid}; # not running
2756         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2757     }
2758
2759     $qmpclient->queue_execute(undef, 2);
2760
2761     foreach my $vmid (keys %$list) {
2762         next if $opt_vmid && ($vmid ne $opt_vmid);
2763         next if !$res->{$vmid}->{pid}; #not running
2764
2765         # we can't use the $qmpclient since it might have already aborted on
2766         # 'query-balloon', but this might also fail for older versions...
2767         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
2768         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
2769     }
2770
2771     foreach my $vmid (keys %$list) {
2772         next if $opt_vmid && ($vmid ne $opt_vmid);
2773         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2774     }
2775
2776     return $res;
2777 }
2778
2779 sub conf_has_serial {
2780     my ($conf) = @_;
2781
2782     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2783         if ($conf->{"serial$i"}) {
2784             return 1;
2785         }
2786     }
2787
2788     return 0;
2789 }
2790
2791 sub conf_has_audio {
2792     my ($conf, $id) = @_;
2793
2794     $id //= 0;
2795     my $audio = $conf->{"audio$id"};
2796     return if !defined($audio);
2797
2798     my $audioproperties = parse_property_string($audio_fmt, $audio);
2799     my $audiodriver = $audioproperties->{driver} // 'spice';
2800
2801     return {
2802         dev => $audioproperties->{device},
2803         dev_id => "audiodev$id",
2804         backend => $audiodriver,
2805         backend_id => "$audiodriver-backend${id}",
2806     };
2807 }
2808
2809 sub audio_devs {
2810     my ($audio, $audiopciaddr, $machine_version) = @_;
2811
2812     my $devs = [];
2813
2814     my $id = $audio->{dev_id};
2815     my $audiodev = "";
2816     if (min_version($machine_version, 4, 2)) {
2817         $audiodev = ",audiodev=$audio->{backend_id}";
2818     }
2819
2820     if ($audio->{dev} eq 'AC97') {
2821         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
2822     } elsif ($audio->{dev} =~ /intel\-hda$/) {
2823         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
2824         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
2825         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
2826     } else {
2827         die "unkown audio device '$audio->{dev}', implement me!";
2828     }
2829
2830     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
2831
2832     return $devs;
2833 }
2834
2835 sub vga_conf_has_spice {
2836     my ($vga) = @_;
2837
2838     my $vgaconf = parse_vga($vga);
2839     my $vgatype = $vgaconf->{type};
2840     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
2841
2842     return $1 || 1;
2843 }
2844
2845 sub is_native($) {
2846     my ($arch) = @_;
2847     return get_host_arch() eq $arch;
2848 }
2849
2850 sub get_vm_arch {
2851     my ($conf) = @_;
2852     return $conf->{arch} // get_host_arch();
2853 }
2854
2855 my $default_machines = {
2856     x86_64 => 'pc',
2857     aarch64 => 'virt',
2858 };
2859
2860 sub get_vm_machine {
2861     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
2862
2863     my $machine = $forcemachine || $conf->{machine};
2864
2865     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
2866         $arch //= 'x86_64';
2867         $machine ||= $default_machines->{$arch};
2868         if ($add_pve_version) {
2869             $kvmversion //= kvm_user_version();
2870             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
2871             $machine .= "+pve$pvever";
2872         }
2873     }
2874
2875     if ($add_pve_version && $machine !~ m/\+pve\d+$/) {
2876         # for version-pinned machines that do not include a pve-version (e.g.
2877         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
2878         $machine .= '+pve0';
2879     }
2880
2881     return $machine;
2882 }
2883
2884 sub get_ovmf_files($) {
2885     my ($arch) = @_;
2886
2887     my $ovmf = $OVMF->{$arch}
2888         or die "no OVMF images known for architecture '$arch'\n";
2889
2890     return @$ovmf;
2891 }
2892
2893 my $Arch2Qemu = {
2894     aarch64 => '/usr/bin/qemu-system-aarch64',
2895     x86_64 => '/usr/bin/qemu-system-x86_64',
2896 };
2897 sub get_command_for_arch($) {
2898     my ($arch) = @_;
2899     return '/usr/bin/kvm' if is_native($arch);
2900
2901     my $cmd = $Arch2Qemu->{$arch}
2902         or die "don't know how to emulate architecture '$arch'\n";
2903     return $cmd;
2904 }
2905
2906 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
2907 # to use in a QEMU command line (-cpu element), first array_intersect the result
2908 # of query_supported_ with query_understood_. This is necessary because:
2909 #
2910 # a) query_understood_ returns flags the host cannot use and
2911 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
2912 #    flags, but CPU settings - with most of them being flags. Those settings
2913 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
2914 #
2915 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
2916 # expensive. If you need the value returned from this, you can get it much
2917 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
2918 # $accel being 'kvm' or 'tcg'.
2919 #
2920 # pvestatd calls this function on startup and whenever the QEMU/KVM version
2921 # changes, automatically populating pmxcfs.
2922 #
2923 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
2924 # since kvm and tcg machines support different flags
2925 #
2926 sub query_supported_cpu_flags {
2927     my ($arch) = @_;
2928
2929     $arch //= get_host_arch();
2930     my $default_machine = $default_machines->{$arch};
2931
2932     my $flags = {};
2933
2934     # FIXME: Once this is merged, the code below should work for ARM as well:
2935     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
2936     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
2937         $arch eq "aarch64";
2938
2939     my $kvm_supported = defined(kvm_version());
2940     my $qemu_cmd = get_command_for_arch($arch);
2941     my $fakevmid = -1;
2942     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
2943
2944     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
2945     my $query_supported_run_qemu = sub {
2946         my ($kvm) = @_;
2947
2948         my $flags = {};
2949         my $cmd = [
2950             $qemu_cmd,
2951             '-machine', $default_machine,
2952             '-display', 'none',
2953             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server,nowait",
2954             '-mon', 'chardev=qmp,mode=control',
2955             '-pidfile', $pidfile,
2956             '-S', '-daemonize'
2957         ];
2958
2959         if (!$kvm) {
2960             push @$cmd, '-accel', 'tcg';
2961         }
2962
2963         my $rc = run_command($cmd, noerr => 1, quiet => 0);
2964         die "QEMU flag querying VM exited with code " . $rc if $rc;
2965
2966         eval {
2967             my $cmd_result = mon_cmd(
2968                 $fakevmid,
2969                 'query-cpu-model-expansion',
2970                 type => 'full',
2971                 model => { name => 'host' }
2972             );
2973
2974             my $props = $cmd_result->{model}->{props};
2975             foreach my $prop (keys %$props) {
2976                 next if $props->{$prop} ne '1';
2977                 # QEMU returns some flags multiple times, with '_', '.' or '-'
2978                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
2979                 # We only keep those with underscores, to match /proc/cpuinfo
2980                 $prop =~ s/\.|-/_/g;
2981                 $flags->{$prop} = 1;
2982             }
2983         };
2984         my $err = $@;
2985
2986         # force stop with 10 sec timeout and 'nocheck'
2987         # always stop, even if QMP failed
2988         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
2989
2990         die $err if $err;
2991
2992         return [ sort keys %$flags ];
2993     };
2994
2995     # We need to query QEMU twice, since KVM and TCG have different supported flags
2996     PVE::QemuConfig->lock_config($fakevmid, sub {
2997         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
2998         warn "warning: failed querying supported tcg flags: $@\n" if $@;
2999
3000         if ($kvm_supported) {
3001             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3002             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3003         }
3004     });
3005
3006     return $flags;
3007 }
3008
3009 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3010 my $understood_cpu_flag_dir = "/usr/share/kvm";
3011 sub query_understood_cpu_flags {
3012     my $arch = get_host_arch();
3013     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3014
3015     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3016         if ! -e $filepath;
3017
3018     my $raw = file_get_contents($filepath);
3019     $raw =~ s/^\s+|\s+$//g;
3020     my @flags = split(/\s+/, $raw);
3021
3022     return \@flags;
3023 }
3024
3025 sub config_to_command {
3026     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu) = @_;
3027
3028     my $cmd = [];
3029     my $globalFlags = [];
3030     my $machineFlags = [];
3031     my $rtcFlags = [];
3032     my $devices = [];
3033     my $pciaddr = '';
3034     my $bridges = {};
3035     my $ostype = $conf->{ostype};
3036     my $winversion = windows_version($ostype);
3037     my $kvm = $conf->{kvm};
3038     my $nodename = nodename();
3039
3040     my $arch = get_vm_arch($conf);
3041     my $kvm_binary = get_command_for_arch($arch);
3042     my $kvmver = kvm_user_version($kvm_binary);
3043
3044     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3045         $kvmver //= "undefined";
3046         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3047     }
3048
3049     my $add_pve_version = min_version($kvmver, 4, 1);
3050
3051     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3052     my $machine_version = extract_version($machine_type, $kvmver);
3053     $kvm //= 1 if is_native($arch);
3054
3055     $machine_version =~ m/(\d+)\.(\d+)/;
3056     my ($machine_major, $machine_minor) = ($1, $2);
3057
3058     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3059         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3060     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3061         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3062             ." please upgrade node '$nodename'\n"
3063     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3064         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3065         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3066             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3067             ." node '$nodename'\n";
3068     }
3069
3070     # if a specific +pve version is required for a feature, use $version_guard
3071     # instead of min_version to allow machines to be run with the minimum
3072     # required version
3073     my $required_pve_version = 0;
3074     my $version_guard = sub {
3075         my ($major, $minor, $pve) = @_;
3076         return 0 if !min_version($machine_version, $major, $minor, $pve);
3077         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3078         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3079         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3080         return 1;
3081     };
3082
3083     if ($kvm && !defined kvm_version()) {
3084         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3085             ." or enable in BIOS.\n";
3086     }
3087
3088     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3089     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3090     my $use_old_bios_files = undef;
3091     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3092
3093     my $cpuunits = defined($conf->{cpuunits}) ?
3094             $conf->{cpuunits} : $defaults->{cpuunits};
3095
3096     push @$cmd, $kvm_binary;
3097
3098     push @$cmd, '-id', $vmid;
3099
3100     my $vmname = $conf->{name} || "vm$vmid";
3101
3102     push @$cmd, '-name', $vmname;
3103
3104     push @$cmd, '-no-shutdown';
3105
3106     my $use_virtio = 0;
3107
3108     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3109     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3110     push @$cmd, '-mon', "chardev=qmp,mode=control";
3111
3112     if (min_version($machine_version, 2, 12)) {
3113         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3114         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3115     }
3116
3117     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3118
3119     push @$cmd, '-daemonize';
3120
3121     if ($conf->{smbios1}) {
3122         my $smbios_conf = parse_smbios1($conf->{smbios1});
3123         if ($smbios_conf->{base64}) {
3124             # Do not pass base64 flag to qemu
3125             delete $smbios_conf->{base64};
3126             my $smbios_string = "";
3127             foreach my $key (keys %$smbios_conf) {
3128                 my $value;
3129                 if ($key eq "uuid") {
3130                     $value = $smbios_conf->{uuid}
3131                 } else {
3132                     $value = decode_base64($smbios_conf->{$key});
3133                 }
3134                 # qemu accepts any binary data, only commas need escaping by double comma
3135                 $value =~ s/,/,,/g;
3136                 $smbios_string .= "," . $key . "=" . $value if $value;
3137             }
3138             push @$cmd, '-smbios', "type=1" . $smbios_string;
3139         } else {
3140             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3141         }
3142     }
3143
3144     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3145         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3146         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3147
3148         my ($path, $format);
3149         if (my $efidisk = $conf->{efidisk0}) {
3150             my $d = parse_drive('efidisk0', $efidisk);
3151             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3152             $format = $d->{format};
3153             if ($storeid) {
3154                 $path = PVE::Storage::path($storecfg, $d->{file});
3155                 if (!defined($format)) {
3156                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3157                     $format = qemu_img_format($scfg, $volname);
3158                 }
3159             } else {
3160                 $path = $d->{file};
3161                 die "efidisk format must be specified\n"
3162                     if !defined($format);
3163             }
3164         } else {
3165             warn "no efidisk configured! Using temporary efivars disk.\n";
3166             $path = "/tmp/$vmid-ovmf.fd";
3167             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3168             $format = 'raw';
3169         }
3170
3171         my $size_str = "";
3172
3173         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3174             $size_str = ",size=" . (-s $ovmf_vars);
3175         }
3176
3177         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3178         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0$size_str,file=$path";
3179     }
3180
3181     # load q35 config
3182     if ($q35) {
3183         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3184         if (min_version($machine_version, 4, 0)) {
3185             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3186         } else {
3187             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3188         }
3189     }
3190
3191     if ($conf->{vmgenid}) {
3192         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3193     }
3194
3195     # add usb controllers
3196     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3197         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3198     push @$devices, @usbcontrollers if @usbcontrollers;
3199     my $vga = parse_vga($conf->{vga});
3200
3201     my $qxlnum = vga_conf_has_spice($conf->{vga});
3202     $vga->{type} = 'qxl' if $qxlnum;
3203
3204     if (!$vga->{type}) {
3205         if ($arch eq 'aarch64') {
3206             $vga->{type} = 'virtio';
3207         } elsif (min_version($machine_version, 2, 9)) {
3208             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3209         } else {
3210             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3211         }
3212     }
3213
3214     # enable absolute mouse coordinates (needed by vnc)
3215     my $tablet;
3216     if (defined($conf->{tablet})) {
3217         $tablet = $conf->{tablet};
3218     } else {
3219         $tablet = $defaults->{tablet};
3220         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3221         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3222     }
3223
3224     if ($tablet) {
3225         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3226         my $kbd = print_keyboarddevice_full($conf, $arch);
3227         push @$devices, '-device', $kbd if defined($kbd);
3228     }
3229
3230     my $bootorder = device_bootorder($conf);
3231
3232     # host pci device passthrough
3233     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3234         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3235
3236     # usb devices
3237     my $usb_dev_features = {};
3238     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3239
3240     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3241         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3242     push @$devices, @usbdevices if @usbdevices;
3243
3244     # serial devices
3245     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3246         if (my $path = $conf->{"serial$i"}) {
3247             if ($path eq 'socket') {
3248                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3249                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3250                 # On aarch64, serial0 is the UART device. Qemu only allows
3251                 # connecting UART devices via the '-serial' command line, as
3252                 # the device has a fixed slot on the hardware...
3253                 if ($arch eq 'aarch64' && $i == 0) {
3254                     push @$devices, '-serial', "chardev:serial$i";
3255                 } else {
3256                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3257                 }
3258             } else {
3259                 die "no such serial device\n" if ! -c $path;
3260                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3261                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3262             }
3263         }
3264     }
3265
3266     # parallel devices
3267     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3268         if (my $path = $conf->{"parallel$i"}) {
3269             die "no such parallel device\n" if ! -c $path;
3270             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3271             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3272             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3273         }
3274     }
3275
3276     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3277         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3278         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3279         push @$devices, @$audio_devs;
3280     }
3281
3282     my $sockets = 1;
3283     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3284     $sockets = $conf->{sockets} if  $conf->{sockets};
3285
3286     my $cores = $conf->{cores} || 1;
3287
3288     my $maxcpus = $sockets * $cores;
3289
3290     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3291
3292     my $allowed_vcpus = $cpuinfo->{cpus};
3293
3294     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3295         if ($allowed_vcpus < $maxcpus);
3296
3297     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3298
3299         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3300         for (my $i = 2; $i <= $vcpus; $i++)  {
3301             my $cpustr = print_cpu_device($conf,$i);
3302             push @$cmd, '-device', $cpustr;
3303         }
3304
3305     } else {
3306
3307         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3308     }
3309     push @$cmd, '-nodefaults';
3310
3311     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3312
3313     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3314
3315     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3316
3317     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3318         push @$devices, '-device', print_vga_device(
3319             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3320         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3321         push @$cmd,  '-vnc', "unix:$socket,password";
3322     } else {
3323         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3324         push @$cmd, '-nographic';
3325     }
3326
3327     # time drift fix
3328     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3329     my $useLocaltime = $conf->{localtime};
3330
3331     if ($winversion >= 5) { # windows
3332         $useLocaltime = 1 if !defined($conf->{localtime});
3333
3334         # use time drift fix when acpi is enabled
3335         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3336             $tdf = 1 if !defined($conf->{tdf});
3337         }
3338     }
3339
3340     if ($winversion >= 6) {
3341         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3342         push @$cmd, '-no-hpet';
3343     }
3344
3345     push @$rtcFlags, 'driftfix=slew' if $tdf;
3346
3347     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3348         push @$rtcFlags, "base=$conf->{startdate}";
3349     } elsif ($useLocaltime) {
3350         push @$rtcFlags, 'base=localtime';
3351     }
3352
3353     if ($forcecpu) {
3354         push @$cmd, '-cpu', $forcecpu;
3355     } else {
3356         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3357     }
3358
3359     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3360
3361     push @$cmd, '-S' if $conf->{freeze};
3362
3363     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3364
3365     my $guest_agent = parse_guest_agent($conf);
3366
3367     if ($guest_agent->{enabled}) {
3368         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3369         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3370
3371         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3372             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3373             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3374             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3375         } elsif ($guest_agent->{type} eq 'isa') {
3376             push @$devices, '-device', "isa-serial,chardev=qga0";
3377         }
3378     }
3379
3380     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3381     if ($rng && $version_guard->(4, 1, 2)) {
3382         check_rng_source($rng->{source});
3383
3384         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3385         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3386         my $limiter_str = "";
3387         if ($max_bytes) {
3388             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3389         }
3390
3391         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3392         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3393         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3394     }
3395
3396     my $spice_port;
3397
3398     if ($qxlnum) {
3399         if ($qxlnum > 1) {
3400             if ($winversion){
3401                 for (my $i = 1; $i < $qxlnum; $i++){
3402                     push @$devices, '-device', print_vga_device(
3403                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3404                 }
3405             } else {
3406                 # assume other OS works like Linux
3407                 my ($ram, $vram) = ("134217728", "67108864");
3408                 if ($vga->{memory}) {
3409                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3410                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3411                 }
3412                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3413                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3414             }
3415         }
3416
3417         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3418
3419         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3420         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3421         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3422
3423         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3424         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3425         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3426
3427         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3428         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3429
3430         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3431         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3432         if ($spice_enhancement->{foldersharing}) {
3433             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3434             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3435         }
3436
3437         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3438         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3439             if $spice_enhancement->{videostreaming};
3440
3441         push @$devices, '-spice', "$spice_opts";
3442     }
3443
3444     # enable balloon by default, unless explicitly disabled
3445     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3446         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3447         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3448     }
3449
3450     if ($conf->{watchdog}) {
3451         my $wdopts = parse_watchdog($conf->{watchdog});
3452         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3453         my $watchdog = $wdopts->{model} || 'i6300esb';
3454         push @$devices, '-device', "$watchdog$pciaddr";
3455         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3456     }
3457
3458     my $vollist = [];
3459     my $scsicontroller = {};
3460     my $ahcicontroller = {};
3461     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3462
3463     # Add iscsi initiator name if available
3464     if (my $initiator = get_initiator_name()) {
3465         push @$devices, '-iscsi', "initiator-name=$initiator";
3466     }
3467
3468     PVE::QemuConfig->foreach_volume($conf, sub {
3469         my ($ds, $drive) = @_;
3470
3471         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3472             push @$vollist, $drive->{file};
3473         }
3474
3475         # ignore efidisk here, already added in bios/fw handling code above
3476         return if $drive->{interface} eq 'efidisk';
3477
3478         $use_virtio = 1 if $ds =~ m/^virtio/;
3479
3480         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3481
3482         if ($drive->{interface} eq 'virtio'){
3483            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3484         }
3485
3486         if ($drive->{interface} eq 'scsi') {
3487
3488             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3489
3490             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3491                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3492
3493             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3494             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3495
3496             my $iothread = '';
3497             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3498                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3499                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3500             } elsif ($drive->{iothread}) {
3501                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3502             }
3503
3504             my $queues = '';
3505             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3506                 $queues = ",num_queues=$drive->{queues}";
3507             }
3508
3509             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3510                 if !$scsicontroller->{$controller};
3511             $scsicontroller->{$controller}=1;
3512         }
3513
3514         if ($drive->{interface} eq 'sata') {
3515             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3516             $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3517             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3518                 if !$ahcicontroller->{$controller};
3519             $ahcicontroller->{$controller}=1;
3520         }
3521
3522         my $drive_cmd = print_drive_commandline_full($storecfg, $vmid, $drive);
3523         $drive_cmd .= ',readonly' if PVE::QemuConfig->is_template($conf);
3524
3525         push @$devices, '-drive',$drive_cmd;
3526         push @$devices, '-device', print_drivedevice_full(
3527             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3528     });
3529
3530     for (my $i = 0; $i < $MAX_NETS; $i++) {
3531         my $netname = "net$i";
3532
3533         next if !$conf->{$netname};
3534         my $d = parse_net($conf->{$netname});
3535         next if !$d;
3536
3537         $use_virtio = 1 if $d->{model} eq 'virtio';
3538
3539         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
3540
3541         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
3542         push @$devices, '-netdev', $netdevfull;
3543
3544         my $netdevicefull = print_netdevice_full(
3545             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
3546
3547         push @$devices, '-device', $netdevicefull;
3548     }
3549
3550     if ($conf->{ivshmem}) {
3551         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3552
3553         my $bus;
3554         if ($q35) {
3555             $bus = print_pcie_addr("ivshmem");
3556         } else {
3557             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3558         }
3559
3560         my $ivshmem_name = $ivshmem->{name} // $vmid;
3561         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3562
3563         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3564         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
3565             .",size=$ivshmem->{size}M";
3566     }
3567
3568     # pci.4 is nested in pci.1
3569     $bridges->{1} = 1 if $bridges->{4};
3570
3571     if (!$q35) {
3572         # add pci bridges
3573         if (min_version($machine_version, 2, 3)) {
3574            $bridges->{1} = 1;
3575            $bridges->{2} = 1;
3576         }
3577
3578         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3579
3580     }
3581
3582     for my $k (sort {$b cmp $a} keys %$bridges) {
3583         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3584
3585         my $k_name = $k;
3586         if ($k == 2 && $legacy_igd) {
3587             $k_name = "$k-igd";
3588         }
3589         $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
3590
3591         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3592         if ($q35) {
3593             # add after -readconfig pve-q35.cfg
3594             splice @$devices, 2, 0, '-device', $devstr;
3595         } else {
3596             unshift @$devices, '-device', $devstr if $k > 0;
3597         }
3598     }
3599
3600     if (!$kvm) {
3601         push @$machineFlags, 'accel=tcg';
3602     }
3603
3604     my $machine_type_min = $machine_type;
3605     if ($add_pve_version) {
3606         $machine_type_min =~ s/\+pve\d+$//;
3607         $machine_type_min .= "+pve$required_pve_version";
3608     }
3609     push @$machineFlags, "type=${machine_type_min}";
3610
3611     push @$cmd, @$devices;
3612     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
3613     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
3614     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
3615
3616     if (my $vmstate = $conf->{vmstate}) {
3617         my $statepath = PVE::Storage::path($storecfg, $vmstate);
3618         push @$vollist, $vmstate;
3619         push @$cmd, '-loadstate', $statepath;
3620         print "activating and using '$vmstate' as vmstate\n";
3621     }
3622
3623     # add custom args
3624     if ($conf->{args}) {
3625         my $aa = PVE::Tools::split_args($conf->{args});
3626         push @$cmd, @$aa;
3627     }
3628
3629     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3630 }
3631
3632 sub check_rng_source {
3633     my ($source) = @_;
3634
3635     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3636     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
3637         if ! -e $source;
3638
3639     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
3640     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
3641         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
3642         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
3643         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
3644             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
3645             ." to the host.\n";
3646     }
3647 }
3648
3649 sub spice_port {
3650     my ($vmid) = @_;
3651
3652     my $res = mon_cmd($vmid, 'query-spice');
3653
3654     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3655 }
3656
3657 sub vm_devices_list {
3658     my ($vmid) = @_;
3659
3660     my $res = mon_cmd($vmid, 'query-pci');
3661     my $devices_to_check = [];
3662     my $devices = {};
3663     foreach my $pcibus (@$res) {
3664         push @$devices_to_check, @{$pcibus->{devices}},
3665     }
3666
3667     while (@$devices_to_check) {
3668         my $to_check = [];
3669         for my $d (@$devices_to_check) {
3670             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3671             next if !$d->{'pci_bridge'};
3672
3673             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3674             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3675         }
3676         $devices_to_check = $to_check;
3677     }
3678
3679     my $resblock = mon_cmd($vmid, 'query-block');
3680     foreach my $block (@$resblock) {
3681         if($block->{device} =~ m/^drive-(\S+)/){
3682                 $devices->{$1} = 1;
3683         }
3684     }
3685
3686     my $resmice = mon_cmd($vmid, 'query-mice');
3687     foreach my $mice (@$resmice) {
3688         if ($mice->{name} eq 'QEMU HID Tablet') {
3689             $devices->{tablet} = 1;
3690             last;
3691         }
3692     }
3693
3694     # for usb devices there is no query-usb
3695     # but we can iterate over the entries in
3696     # qom-list path=/machine/peripheral
3697     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3698     foreach my $per (@$resperipheral) {
3699         if ($per->{name} =~ m/^usb\d+$/) {
3700             $devices->{$per->{name}} = 1;
3701         }
3702     }
3703
3704     return $devices;
3705 }
3706
3707 sub vm_deviceplug {
3708     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3709
3710     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3711
3712     my $devices_list = vm_devices_list($vmid);
3713     return 1 if defined($devices_list->{$deviceid});
3714
3715     # add PCI bridge if we need it for the device
3716     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
3717
3718     if ($deviceid eq 'tablet') {
3719
3720         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
3721
3722     } elsif ($deviceid eq 'keyboard') {
3723
3724         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
3725
3726     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3727
3728         die "usb hotplug currently not reliable\n";
3729         # since we can't reliably hot unplug all added usb devices and usb
3730         # passthrough breaks live migration we disable usb hotplugging for now
3731         #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3732
3733     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3734
3735         qemu_iothread_add($vmid, $deviceid, $device);
3736
3737         qemu_driveadd($storecfg, $vmid, $device);
3738         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3739
3740         qemu_deviceadd($vmid, $devicefull);
3741         eval { qemu_deviceaddverify($vmid, $deviceid); };
3742         if (my $err = $@) {
3743             eval { qemu_drivedel($vmid, $deviceid); };
3744             warn $@ if $@;
3745             die $err;
3746         }
3747
3748     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3749
3750
3751         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3752         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3753         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3754
3755         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3756
3757         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3758             qemu_iothread_add($vmid, $deviceid, $device);
3759             $devicefull .= ",iothread=iothread-$deviceid";
3760         }
3761
3762         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3763             $devicefull .= ",num_queues=$device->{queues}";
3764         }
3765
3766         qemu_deviceadd($vmid, $devicefull);
3767         qemu_deviceaddverify($vmid, $deviceid);
3768
3769     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3770
3771         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
3772         qemu_driveadd($storecfg, $vmid, $device);
3773
3774         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3775         eval { qemu_deviceadd($vmid, $devicefull); };
3776         if (my $err = $@) {
3777             eval { qemu_drivedel($vmid, $deviceid); };
3778             warn $@ if $@;
3779             die $err;
3780         }
3781
3782     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3783
3784         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
3785
3786         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
3787         my $use_old_bios_files = undef;
3788         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3789
3790         my $netdevicefull = print_netdevice_full(
3791             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
3792         qemu_deviceadd($vmid, $netdevicefull);
3793         eval {
3794             qemu_deviceaddverify($vmid, $deviceid);
3795             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
3796         };
3797         if (my $err = $@) {
3798             eval { qemu_netdevdel($vmid, $deviceid); };
3799             warn $@ if $@;
3800             die $err;
3801         }
3802
3803     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3804
3805         my $bridgeid = $2;
3806         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3807         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3808
3809         qemu_deviceadd($vmid, $devicefull);
3810         qemu_deviceaddverify($vmid, $deviceid);
3811
3812     } else {
3813         die "can't hotplug device '$deviceid'\n";
3814     }
3815
3816     return 1;
3817 }
3818
3819 # fixme: this should raise exceptions on error!
3820 sub vm_deviceunplug {
3821     my ($vmid, $conf, $deviceid) = @_;
3822
3823     my $devices_list = vm_devices_list($vmid);
3824     return 1 if !defined($devices_list->{$deviceid});
3825
3826     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
3827     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
3828
3829     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
3830
3831         qemu_devicedel($vmid, $deviceid);
3832
3833     } elsif ($deviceid =~ m/^usb\d+$/) {
3834
3835         die "usb hotplug currently not reliable\n";
3836         # when unplugging usb devices this way, there may be remaining usb
3837         # controllers/hubs so we disable it for now
3838         #qemu_devicedel($vmid, $deviceid);
3839         #qemu_devicedelverify($vmid, $deviceid);
3840
3841     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3842
3843         qemu_devicedel($vmid, $deviceid);
3844         qemu_devicedelverify($vmid, $deviceid);
3845         qemu_drivedel($vmid, $deviceid);
3846         qemu_iothread_del($conf, $vmid, $deviceid);
3847
3848     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3849
3850         qemu_devicedel($vmid, $deviceid);
3851         qemu_devicedelverify($vmid, $deviceid);
3852         qemu_iothread_del($conf, $vmid, $deviceid);
3853
3854     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3855
3856         qemu_devicedel($vmid, $deviceid);
3857         qemu_drivedel($vmid, $deviceid);
3858         qemu_deletescsihw($conf, $vmid, $deviceid);
3859
3860     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3861
3862         qemu_devicedel($vmid, $deviceid);
3863         qemu_devicedelverify($vmid, $deviceid);
3864         qemu_netdevdel($vmid, $deviceid);
3865
3866     } else {
3867         die "can't unplug device '$deviceid'\n";
3868     }
3869
3870     return 1;
3871 }
3872
3873 sub qemu_deviceadd {
3874     my ($vmid, $devicefull) = @_;
3875
3876     $devicefull = "driver=".$devicefull;
3877     my %options =  split(/[=,]/, $devicefull);
3878
3879     mon_cmd($vmid, "device_add" , %options);
3880 }
3881
3882 sub qemu_devicedel {
3883     my ($vmid, $deviceid) = @_;
3884
3885     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
3886 }
3887
3888 sub qemu_iothread_add {
3889     my($vmid, $deviceid, $device) = @_;
3890
3891     if ($device->{iothread}) {
3892         my $iothreads = vm_iothreads_list($vmid);
3893         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3894     }
3895 }
3896
3897 sub qemu_iothread_del {
3898     my($conf, $vmid, $deviceid) = @_;
3899
3900     my $confid = $deviceid;
3901     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
3902         $confid = 'scsi' . $1;
3903     }
3904     my $device = parse_drive($confid, $conf->{$confid});
3905     if ($device->{iothread}) {
3906         my $iothreads = vm_iothreads_list($vmid);
3907         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3908     }
3909 }
3910
3911 sub qemu_objectadd {
3912     my($vmid, $objectid, $qomtype) = @_;
3913
3914     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3915
3916     return 1;
3917 }
3918
3919 sub qemu_objectdel {
3920     my($vmid, $objectid) = @_;
3921
3922     mon_cmd($vmid, "object-del", id => $objectid);
3923
3924     return 1;
3925 }
3926
3927 sub qemu_driveadd {
3928     my ($storecfg, $vmid, $device) = @_;
3929
3930     my $drive = print_drive_commandline_full($storecfg, $vmid, $device);
3931     $drive =~ s/\\/\\\\/g;
3932     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
3933
3934     # If the command succeeds qemu prints: "OK"
3935     return 1 if $ret =~ m/OK/s;
3936
3937     die "adding drive failed: $ret\n";
3938 }
3939
3940 sub qemu_drivedel {
3941     my($vmid, $deviceid) = @_;
3942
3943     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
3944     $ret =~ s/^\s+//;
3945
3946     return 1 if $ret eq "";
3947
3948     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3949     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3950
3951     die "deleting drive $deviceid failed : $ret\n";
3952 }
3953
3954 sub qemu_deviceaddverify {
3955     my ($vmid, $deviceid) = @_;
3956
3957     for (my $i = 0; $i <= 5; $i++) {
3958          my $devices_list = vm_devices_list($vmid);
3959          return 1 if defined($devices_list->{$deviceid});
3960          sleep 1;
3961     }
3962
3963     die "error on hotplug device '$deviceid'\n";
3964 }
3965
3966
3967 sub qemu_devicedelverify {
3968     my ($vmid, $deviceid) = @_;
3969
3970     # need to verify that the device is correctly removed as device_del
3971     # is async and empty return is not reliable
3972
3973     for (my $i = 0; $i <= 5; $i++) {
3974          my $devices_list = vm_devices_list($vmid);
3975          return 1 if !defined($devices_list->{$deviceid});
3976          sleep 1;
3977     }
3978
3979     die "error on hot-unplugging device '$deviceid'\n";
3980 }
3981
3982 sub qemu_findorcreatescsihw {
3983     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
3984
3985     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3986
3987     my $scsihwid="$controller_prefix$controller";
3988     my $devices_list = vm_devices_list($vmid);
3989
3990     if(!defined($devices_list->{$scsihwid})) {
3991         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
3992     }
3993
3994     return 1;
3995 }
3996
3997 sub qemu_deletescsihw {
3998     my ($conf, $vmid, $opt) = @_;
3999
4000     my $device = parse_drive($opt, $conf->{$opt});
4001
4002     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4003         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4004         return 1;
4005     }
4006
4007     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4008
4009     my $devices_list = vm_devices_list($vmid);
4010     foreach my $opt (keys %{$devices_list}) {
4011         if (is_valid_drivename($opt)) {
4012             my $drive = parse_drive($opt, $conf->{$opt});
4013             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4014                 return 1;
4015             }
4016         }
4017     }
4018
4019     my $scsihwid="scsihw$controller";
4020
4021     vm_deviceunplug($vmid, $conf, $scsihwid);
4022
4023     return 1;
4024 }
4025
4026 sub qemu_add_pci_bridge {
4027     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4028
4029     my $bridges = {};
4030
4031     my $bridgeid;
4032
4033     print_pci_addr($device, $bridges, $arch, $machine_type);
4034
4035     while (my ($k, $v) = each %$bridges) {
4036         $bridgeid = $k;
4037     }
4038     return 1 if !defined($bridgeid) || $bridgeid < 1;
4039
4040     my $bridge = "pci.$bridgeid";
4041     my $devices_list = vm_devices_list($vmid);
4042
4043     if (!defined($devices_list->{$bridge})) {
4044         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4045     }
4046
4047     return 1;
4048 }
4049
4050 sub qemu_set_link_status {
4051     my ($vmid, $device, $up) = @_;
4052
4053     mon_cmd($vmid, "set_link", name => $device,
4054                up => $up ? JSON::true : JSON::false);
4055 }
4056
4057 sub qemu_netdevadd {
4058     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4059
4060     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4061     my %options =  split(/[=,]/, $netdev);
4062
4063     if (defined(my $vhost = $options{vhost})) {
4064         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4065     }
4066
4067     if (defined(my $queues = $options{queues})) {
4068         $options{queues} = $queues + 0;
4069     }
4070
4071     mon_cmd($vmid, "netdev_add",  %options);
4072     return 1;
4073 }
4074
4075 sub qemu_netdevdel {
4076     my ($vmid, $deviceid) = @_;
4077
4078     mon_cmd($vmid, "netdev_del", id => $deviceid);
4079 }
4080
4081 sub qemu_usb_hotplug {
4082     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4083
4084     return if !$device;
4085
4086     # remove the old one first
4087     vm_deviceunplug($vmid, $conf, $deviceid);
4088
4089     # check if xhci controller is necessary and available
4090     if ($device->{usb3}) {
4091
4092         my $devicelist = vm_devices_list($vmid);
4093
4094         if (!$devicelist->{xhci}) {
4095             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4096             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4097         }
4098     }
4099     my $d = parse_usb_device($device->{host});
4100     $d->{usb3} = $device->{usb3};
4101
4102     # add the new one
4103     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4104 }
4105
4106 sub qemu_cpu_hotplug {
4107     my ($vmid, $conf, $vcpus) = @_;
4108
4109     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4110
4111     my $sockets = 1;
4112     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4113     $sockets = $conf->{sockets} if  $conf->{sockets};
4114     my $cores = $conf->{cores} || 1;
4115     my $maxcpus = $sockets * $cores;
4116
4117     $vcpus = $maxcpus if !$vcpus;
4118
4119     die "you can't add more vcpus than maxcpus\n"
4120         if $vcpus > $maxcpus;
4121
4122     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4123
4124     if ($vcpus < $currentvcpus) {
4125
4126         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4127
4128             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4129                 qemu_devicedel($vmid, "cpu$i");
4130                 my $retry = 0;
4131                 my $currentrunningvcpus = undef;
4132                 while (1) {
4133                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4134                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4135                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4136                     $retry++;
4137                     sleep 1;
4138                 }
4139                 #update conf after each succesfull cpu unplug
4140                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4141                 PVE::QemuConfig->write_config($vmid, $conf);
4142             }
4143         } else {
4144             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4145         }
4146
4147         return;
4148     }
4149
4150     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4151     die "vcpus in running vm does not match its configuration\n"
4152         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4153
4154     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4155
4156         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4157             my $cpustr = print_cpu_device($conf, $i);
4158             qemu_deviceadd($vmid, $cpustr);
4159
4160             my $retry = 0;
4161             my $currentrunningvcpus = undef;
4162             while (1) {
4163                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4164                 last if scalar(@{$currentrunningvcpus}) == $i;
4165                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4166                 sleep 1;
4167                 $retry++;
4168             }
4169             #update conf after each succesfull cpu hotplug
4170             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4171             PVE::QemuConfig->write_config($vmid, $conf);
4172         }
4173     } else {
4174
4175         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4176             mon_cmd($vmid, "cpu-add", id => int($i));
4177         }
4178     }
4179 }
4180
4181 sub qemu_block_set_io_throttle {
4182     my ($vmid, $deviceid,
4183         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4184         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4185         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4186         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4187
4188     return if !check_running($vmid) ;
4189
4190     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4191         bps => int($bps),
4192         bps_rd => int($bps_rd),
4193         bps_wr => int($bps_wr),
4194         iops => int($iops),
4195         iops_rd => int($iops_rd),
4196         iops_wr => int($iops_wr),
4197         bps_max => int($bps_max),
4198         bps_rd_max => int($bps_rd_max),
4199         bps_wr_max => int($bps_wr_max),
4200         iops_max => int($iops_max),
4201         iops_rd_max => int($iops_rd_max),
4202         iops_wr_max => int($iops_wr_max),
4203         bps_max_length => int($bps_max_length),
4204         bps_rd_max_length => int($bps_rd_max_length),
4205         bps_wr_max_length => int($bps_wr_max_length),
4206         iops_max_length => int($iops_max_length),
4207         iops_rd_max_length => int($iops_rd_max_length),
4208         iops_wr_max_length => int($iops_wr_max_length),
4209     );
4210
4211 }
4212
4213 sub qemu_block_resize {
4214     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4215
4216     my $running = check_running($vmid);
4217
4218     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4219
4220     return if !$running;
4221
4222     my $padding = (1024 - $size % 1024) % 1024;
4223     $size = $size + $padding;
4224
4225     mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4226
4227 }
4228
4229 sub qemu_volume_snapshot {
4230     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4231
4232     my $running = check_running($vmid);
4233
4234     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4235         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4236     } else {
4237         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4238     }
4239 }
4240
4241 sub qemu_volume_snapshot_delete {
4242     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4243
4244     my $running = check_running($vmid);
4245
4246     if($running) {
4247
4248         $running = undef;
4249         my $conf = PVE::QemuConfig->load_config($vmid);
4250         PVE::QemuConfig->foreach_volume($conf, sub {
4251             my ($ds, $drive) = @_;
4252             $running = 1 if $drive->{file} eq $volid;
4253         });
4254     }
4255
4256     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4257         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4258     } else {
4259         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4260     }
4261 }
4262
4263 sub set_migration_caps {
4264     my ($vmid) = @_;
4265
4266     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4267
4268     my $cap_ref = [];
4269
4270     my $enabled_cap = {
4271         "auto-converge" => 1,
4272         "xbzrle" => 1,
4273         "x-rdma-pin-all" => 0,
4274         "zero-blocks" => 0,
4275         "compress" => 0,
4276         "dirty-bitmaps" => $qemu_support->{'pbs-dirty-bitmap-migration'} ? 1 : 0,
4277     };
4278
4279     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4280
4281     for my $supported_capability (@$supported_capabilities) {
4282         push @$cap_ref, {
4283             capability => $supported_capability->{capability},
4284             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4285         };
4286     }
4287
4288     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4289 }
4290
4291 sub foreach_volid {
4292     my ($conf, $func, @param) = @_;
4293
4294     my $volhash = {};
4295
4296     my $test_volid = sub {
4297         my ($key, $drive, $snapname) = @_;
4298
4299         my $volid = $drive->{file};
4300         return if !$volid;
4301
4302         $volhash->{$volid}->{cdrom} //= 1;
4303         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4304
4305         my $replicate = $drive->{replicate} // 1;
4306         $volhash->{$volid}->{replicate} //= 0;
4307         $volhash->{$volid}->{replicate} = 1 if $replicate;
4308
4309         $volhash->{$volid}->{shared} //= 0;
4310         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4311
4312         $volhash->{$volid}->{referenced_in_config} //= 0;
4313         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4314
4315         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4316             if defined($snapname);
4317
4318         my $size = $drive->{size};
4319         $volhash->{$volid}->{size} //= $size if $size;
4320
4321         $volhash->{$volid}->{is_vmstate} //= 0;
4322         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4323
4324         $volhash->{$volid}->{is_unused} //= 0;
4325         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4326     };
4327
4328     my $include_opts = {
4329         extra_keys => ['vmstate'],
4330         include_unused => 1,
4331     };
4332
4333     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4334     foreach my $snapname (keys %{$conf->{snapshots}}) {
4335         my $snap = $conf->{snapshots}->{$snapname};
4336         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4337     }
4338
4339     foreach my $volid (keys %$volhash) {
4340         &$func($volid, $volhash->{$volid}, @param);
4341     }
4342 }
4343
4344 my $fast_plug_option = {
4345     'lock' => 1,
4346     'name' => 1,
4347     'onboot' => 1,
4348     'shares' => 1,
4349     'startup' => 1,
4350     'description' => 1,
4351     'protection' => 1,
4352     'vmstatestorage' => 1,
4353     'hookscript' => 1,
4354     'tags' => 1,
4355 };
4356
4357 # hotplug changes in [PENDING]
4358 # $selection hash can be used to only apply specified options, for
4359 # example: { cores => 1 } (only apply changed 'cores')
4360 # $errors ref is used to return error messages
4361 sub vmconfig_hotplug_pending {
4362     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4363
4364     my $defaults = load_defaults();
4365     my $arch = get_vm_arch($conf);
4366     my $machine_type = get_vm_machine($conf, undef, $arch);
4367
4368     # commit values which do not have any impact on running VM first
4369     # Note: those option cannot raise errors, we we do not care about
4370     # $selection and always apply them.
4371
4372     my $add_error = sub {
4373         my ($opt, $msg) = @_;
4374         $errors->{$opt} = "hotplug problem - $msg";
4375     };
4376
4377     my $changes = 0;
4378     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4379         if ($fast_plug_option->{$opt}) {
4380             $conf->{$opt} = $conf->{pending}->{$opt};
4381             delete $conf->{pending}->{$opt};
4382             $changes = 1;
4383         }
4384     }
4385
4386     if ($changes) {
4387         PVE::QemuConfig->write_config($vmid, $conf);
4388     }
4389
4390     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4391
4392     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4393     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4394     foreach my $opt (sort keys %$pending_delete_hash) {
4395         next if $selection && !$selection->{$opt};
4396         my $force = $pending_delete_hash->{$opt}->{force};
4397         eval {
4398             if ($opt eq 'hotplug') {
4399                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4400             } elsif ($opt eq 'tablet') {
4401                 die "skip\n" if !$hotplug_features->{usb};
4402                 if ($defaults->{tablet}) {
4403                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4404                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4405                         if $arch eq 'aarch64';
4406                 } else {
4407                     vm_deviceunplug($vmid, $conf, 'tablet');
4408                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4409                 }
4410             } elsif ($opt =~ m/^usb\d+/) {
4411                 die "skip\n";
4412                 # since we cannot reliably hot unplug usb devices we are disabling it
4413                 #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4414                 #vm_deviceunplug($vmid, $conf, $opt);
4415             } elsif ($opt eq 'vcpus') {
4416                 die "skip\n" if !$hotplug_features->{cpu};
4417                 qemu_cpu_hotplug($vmid, $conf, undef);
4418             } elsif ($opt eq 'balloon') {
4419                 # enable balloon device is not hotpluggable
4420                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4421                 # here we reset the ballooning value to memory
4422                 my $balloon = $conf->{memory} || $defaults->{memory};
4423                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4424             } elsif ($fast_plug_option->{$opt}) {
4425                 # do nothing
4426             } elsif ($opt =~ m/^net(\d+)$/) {
4427                 die "skip\n" if !$hotplug_features->{network};
4428                 vm_deviceunplug($vmid, $conf, $opt);
4429             } elsif (is_valid_drivename($opt)) {
4430                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4431                 vm_deviceunplug($vmid, $conf, $opt);
4432                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4433             } elsif ($opt =~ m/^memory$/) {
4434                 die "skip\n" if !$hotplug_features->{memory};
4435                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4436             } elsif ($opt eq 'cpuunits') {
4437                 $cgroup->change_cpu_shares(undef, $defaults->{cpuunits});
4438             } elsif ($opt eq 'cpulimit') {
4439                 $cgroup->change_cpu_quota(-1, 100000);
4440             } else {
4441                 die "skip\n";
4442             }
4443         };
4444         if (my $err = $@) {
4445             &$add_error($opt, $err) if $err ne "skip\n";
4446         } else {
4447             delete $conf->{$opt};
4448             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4449         }
4450     }
4451
4452     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4453     $apply_pending_cloudinit = sub {
4454         return if $apply_pending_cloudinit_done; # once is enough
4455         $apply_pending_cloudinit_done = 1; # once is enough
4456
4457         my ($key, $value) = @_;
4458
4459         my @cloudinit_opts = keys %$confdesc_cloudinit;
4460         foreach my $opt (keys %{$conf->{pending}}) {
4461             next if !grep { $_ eq $opt } @cloudinit_opts;
4462             $conf->{$opt} = delete $conf->{pending}->{$opt};
4463         }
4464
4465         my $new_conf = { %$conf };
4466         $new_conf->{$key} = $value;
4467         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4468     };
4469
4470     foreach my $opt (keys %{$conf->{pending}}) {
4471         next if $selection && !$selection->{$opt};
4472         my $value = $conf->{pending}->{$opt};
4473         eval {
4474             if ($opt eq 'hotplug') {
4475                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4476             } elsif ($opt eq 'tablet') {
4477                 die "skip\n" if !$hotplug_features->{usb};
4478                 if ($value == 1) {
4479                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4480                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4481                         if $arch eq 'aarch64';
4482                 } elsif ($value == 0) {
4483                     vm_deviceunplug($vmid, $conf, 'tablet');
4484                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4485                 }
4486             } elsif ($opt =~ m/^usb\d+$/) {
4487                 die "skip\n";
4488                 # since we cannot reliably hot unplug usb devices we disable it for now
4489                 #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4490                 #my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4491                 #die "skip\n" if !$d;
4492                 #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4493             } elsif ($opt eq 'vcpus') {
4494                 die "skip\n" if !$hotplug_features->{cpu};
4495                 qemu_cpu_hotplug($vmid, $conf, $value);
4496             } elsif ($opt eq 'balloon') {
4497                 # enable/disable balloning device is not hotpluggable
4498                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4499                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4500                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4501
4502                 # allow manual ballooning if shares is set to zero
4503                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4504                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4505                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4506                 }
4507             } elsif ($opt =~ m/^net(\d+)$/) {
4508                 # some changes can be done without hotplug
4509                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4510                                     $vmid, $opt, $value, $arch, $machine_type);
4511             } elsif (is_valid_drivename($opt)) {
4512                 die "skip\n" if $opt eq 'efidisk0';
4513                 # some changes can be done without hotplug
4514                 my $drive = parse_drive($opt, $value);
4515                 if (drive_is_cloudinit($drive)) {
4516                     &$apply_pending_cloudinit($opt, $value);
4517                 }
4518                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4519                                      $vmid, $opt, $value, $arch, $machine_type);
4520             } elsif ($opt =~ m/^memory$/) { #dimms
4521                 die "skip\n" if !$hotplug_features->{memory};
4522                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4523             } elsif ($opt eq 'cpuunits') {
4524                 $cgroup->change_cpu_shares($conf->{pending}->{$opt}, $defaults->{cpuunits});
4525             } elsif ($opt eq 'cpulimit') {
4526                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4527                 $cgroup->change_cpu_quota($cpulimit, 100000);
4528             } else {
4529                 die "skip\n";  # skip non-hot-pluggable options
4530             }
4531         };
4532         if (my $err = $@) {
4533             &$add_error($opt, $err) if $err ne "skip\n";
4534         } else {
4535             $conf->{$opt} = $value;
4536             delete $conf->{pending}->{$opt};
4537         }
4538     }
4539
4540     PVE::QemuConfig->write_config($vmid, $conf);
4541 }
4542
4543 sub try_deallocate_drive {
4544     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4545
4546     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4547         my $volid = $drive->{file};
4548         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4549             my $sid = PVE::Storage::parse_volume_id($volid);
4550             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4551
4552             # check if the disk is really unused
4553             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4554                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4555             PVE::Storage::vdisk_free($storecfg, $volid);
4556             return 1;
4557         } else {
4558             # If vm is not owner of this disk remove from config
4559             return 1;
4560         }
4561     }
4562
4563     return;
4564 }
4565
4566 sub vmconfig_delete_or_detach_drive {
4567     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4568
4569     my $drive = parse_drive($opt, $conf->{$opt});
4570
4571     my $rpcenv = PVE::RPCEnvironment::get();
4572     my $authuser = $rpcenv->get_user();
4573
4574     if ($force) {
4575         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4576         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4577     } else {
4578         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4579     }
4580 }
4581
4582
4583
4584 sub vmconfig_apply_pending {
4585     my ($vmid, $conf, $storecfg, $errors) = @_;
4586
4587     my $add_apply_error = sub {
4588         my ($opt, $msg) = @_;
4589         my $err_msg = "unable to apply pending change $opt : $msg";
4590         $errors->{$opt} = $err_msg;
4591         warn $err_msg;
4592     };
4593
4594     # cold plug
4595
4596     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4597     foreach my $opt (sort keys %$pending_delete_hash) {
4598         my $force = $pending_delete_hash->{$opt}->{force};
4599         eval {
4600             if ($opt =~ m/^unused/) {
4601                 die "internal error";
4602             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4603                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4604             }
4605         };
4606         if (my $err = $@) {
4607             $add_apply_error->($opt, $err);
4608         } else {
4609             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4610             delete $conf->{$opt};
4611         }
4612     }
4613
4614     PVE::QemuConfig->cleanup_pending($conf);
4615
4616     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4617         next if $opt eq 'delete'; # just to be sure
4618         eval {
4619             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4620                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4621             }
4622         };
4623         if (my $err = $@) {
4624             $add_apply_error->($opt, $err);
4625         } else {
4626             $conf->{$opt} = delete $conf->{pending}->{$opt};
4627         }
4628     }
4629
4630     # write all changes at once to avoid unnecessary i/o
4631     PVE::QemuConfig->write_config($vmid, $conf);
4632 }
4633
4634 sub vmconfig_update_net {
4635     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4636
4637     my $newnet = parse_net($value);
4638
4639     if ($conf->{$opt}) {
4640         my $oldnet = parse_net($conf->{$opt});
4641
4642         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4643             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4644             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4645             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4646
4647             # for non online change, we try to hot-unplug
4648             die "skip\n" if !$hotplug;
4649             vm_deviceunplug($vmid, $conf, $opt);
4650         } else {
4651
4652             die "internal error" if $opt !~ m/net(\d+)/;
4653             my $iface = "tap${vmid}i$1";
4654
4655             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4656                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4657                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4658                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4659                 PVE::Network::tap_unplug($iface);
4660
4661                 if ($have_sdn) {
4662                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4663                 } else {
4664                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4665                 }
4666             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4667                 # Rate can be applied on its own but any change above needs to
4668                 # include the rate in tap_plug since OVS resets everything.
4669                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4670             }
4671
4672             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4673                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4674             }
4675
4676             return 1;
4677         }
4678     }
4679
4680     if ($hotplug) {
4681         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4682     } else {
4683         die "skip\n";
4684     }
4685 }
4686
4687 sub vmconfig_update_disk {
4688     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4689
4690     my $drive = parse_drive($opt, $value);
4691
4692     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
4693         my $media = $drive->{media} || 'disk';
4694         my $oldmedia = $old_drive->{media} || 'disk';
4695         die "unable to change media type\n" if $media ne $oldmedia;
4696
4697         if (!drive_is_cdrom($old_drive)) {
4698
4699             if ($drive->{file} ne $old_drive->{file}) {
4700
4701                 die "skip\n" if !$hotplug;
4702
4703                 # unplug and register as unused
4704                 vm_deviceunplug($vmid, $conf, $opt);
4705                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4706
4707             } else {
4708                 # update existing disk
4709
4710                 # skip non hotpluggable value
4711                 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4712                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4713                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4714                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
4715                     safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
4716                     die "skip\n";
4717                 }
4718
4719                 # apply throttle
4720                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4721                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4722                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4723                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4724                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4725                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4726                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4727                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4728                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4729                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4730                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4731                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4732                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4733                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4734                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4735                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4736                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4737                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4738
4739                     qemu_block_set_io_throttle(
4740                         $vmid,"drive-$opt",
4741                         ($drive->{mbps} || 0)*1024*1024,
4742                         ($drive->{mbps_rd} || 0)*1024*1024,
4743                         ($drive->{mbps_wr} || 0)*1024*1024,
4744                          $drive->{iops} || 0,
4745                          $drive->{iops_rd} || 0,
4746                          $drive->{iops_wr} || 0,
4747                         ($drive->{mbps_max} || 0)*1024*1024,
4748                         ($drive->{mbps_rd_max} || 0)*1024*1024,
4749                         ($drive->{mbps_wr_max} || 0)*1024*1024,
4750                          $drive->{iops_max} || 0,
4751                          $drive->{iops_rd_max} || 0,
4752                          $drive->{iops_wr_max} || 0,
4753                          $drive->{bps_max_length} || 1,
4754                          $drive->{bps_rd_max_length} || 1,
4755                          $drive->{bps_wr_max_length} || 1,
4756                          $drive->{iops_max_length} || 1,
4757                          $drive->{iops_rd_max_length} || 1,
4758                          $drive->{iops_wr_max_length} || 1,
4759                     );
4760
4761                 }
4762
4763                 return 1;
4764             }
4765
4766         } else { # cdrom
4767
4768             if ($drive->{file} eq 'none') {
4769                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4770                 if (drive_is_cloudinit($old_drive)) {
4771                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4772                 }
4773             } else {
4774                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4775
4776                 # force eject if locked
4777                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4778
4779                 if ($path) {
4780                     mon_cmd($vmid, "blockdev-change-medium",
4781                         id => "$opt", filename => "$path");
4782                 }
4783             }
4784
4785             return 1;
4786         }
4787     }
4788
4789     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4790     # hotplug new disks
4791     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4792     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
4793 }
4794
4795 # called in locked context by incoming migration
4796 sub vm_migrate_get_nbd_disks {
4797     my ($storecfg, $conf, $replicated_volumes) = @_;
4798
4799     my $local_volumes = {};
4800     PVE::QemuConfig->foreach_volume($conf, sub {
4801         my ($ds, $drive) = @_;
4802
4803         return if drive_is_cdrom($drive);
4804
4805         my $volid = $drive->{file};
4806
4807         return if !$volid;
4808
4809         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4810
4811         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4812         return if $scfg->{shared};
4813
4814         # replicated disks re-use existing state via bitmap
4815         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
4816         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
4817     });
4818     return $local_volumes;
4819 }
4820
4821 # called in locked context by incoming migration
4822 sub vm_migrate_alloc_nbd_disks {
4823     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
4824
4825     my $format = undef;
4826
4827     my $nbd = {};
4828     foreach my $opt (sort keys %$source_volumes) {
4829         my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
4830
4831         if ($use_existing) {
4832             $nbd->{$opt}->{drivestr} = print_drive($drive);
4833             $nbd->{$opt}->{volid} = $volid;
4834             $nbd->{$opt}->{replicated} = 1;
4835             next;
4836         }
4837
4838         # If a remote storage is specified and the format of the original
4839         # volume is not available there, fall back to the default format.
4840         # Otherwise use the same format as the original.
4841         if (!$storagemap->{identity}) {
4842             $storeid = map_storage($storagemap, $storeid);
4843             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4844             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4845             my $fileFormat = qemu_img_format($scfg, $volname);
4846             $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
4847         } else {
4848             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4849             $format = qemu_img_format($scfg, $volname);
4850         }
4851
4852         my $size = $drive->{size} / 1024;
4853         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
4854         my $newdrive = $drive;
4855         $newdrive->{format} = $format;
4856         $newdrive->{file} = $newvolid;
4857         my $drivestr = print_drive($newdrive);
4858         $nbd->{$opt}->{drivestr} = $drivestr;
4859         $nbd->{$opt}->{volid} = $newvolid;
4860     }
4861
4862     return $nbd;
4863 }
4864
4865 # see vm_start_nolock for parameters, additionally:
4866 # migrate_opts:
4867 #   storagemap = parsed storage map for allocating NBD disks
4868 sub vm_start {
4869     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
4870
4871     return PVE::QemuConfig->lock_config($vmid, sub {
4872         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
4873
4874         die "you can't start a vm if it's a template\n"
4875             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
4876
4877         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
4878         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
4879
4880         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
4881
4882         if ($has_backup_lock && $running) {
4883             # a backup is currently running, attempt to start the guest in the
4884             # existing QEMU instance
4885             return vm_resume($vmid);
4886         }
4887
4888         PVE::QemuConfig->check_lock($conf)
4889             if !($params->{skiplock} || $has_suspended_lock);
4890
4891         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
4892
4893         die "VM $vmid already running\n" if $running;
4894
4895         if (my $storagemap = $migrate_opts->{storagemap}) {
4896             my $replicated = $migrate_opts->{replicated_volumes};
4897             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
4898             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
4899
4900             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
4901                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
4902             }
4903         }
4904
4905         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
4906     });
4907 }
4908
4909
4910 # params:
4911 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
4912 #   skiplock => 0/1, skip checking for config lock
4913 #   skiptemplate => 0/1, skip checking whether VM is template
4914 #   forcemachine => to force Qemu machine (rollback/migration)
4915 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
4916 #   timeout => in seconds
4917 #   paused => start VM in paused state (backup)
4918 #   resume => resume from hibernation
4919 # migrate_opts:
4920 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
4921 #   migratedfrom => source node
4922 #   spice_ticket => used for spice migration, passed via tunnel/stdin
4923 #   network => CIDR of migration network
4924 #   type => secure/insecure - tunnel over encrypted connection or plain-text
4925 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
4926 #   replicated_volumes = which volids should be re-used with bitmaps for nbd migration
4927 sub vm_start_nolock {
4928     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
4929
4930     my $statefile = $params->{statefile};
4931     my $resume = $params->{resume};
4932
4933     my $migratedfrom = $migrate_opts->{migratedfrom};
4934     my $migration_type = $migrate_opts->{type};
4935
4936     my $res = {};
4937
4938     # clean up leftover reboot request files
4939     eval { clear_reboot_request($vmid); };
4940     warn $@ if $@;
4941
4942     if (!$statefile && scalar(keys %{$conf->{pending}})) {
4943         vmconfig_apply_pending($vmid, $conf, $storecfg);
4944         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4945     }
4946
4947     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4948
4949     my $defaults = load_defaults();
4950
4951     # set environment variable useful inside network script
4952     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4953
4954     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
4955
4956     my $forcemachine = $params->{forcemachine};
4957     my $forcecpu = $params->{forcecpu};
4958     if ($resume) {
4959         # enforce machine and CPU type on suspended vm to ensure HW compatibility
4960         $forcemachine = $conf->{runningmachine};
4961         $forcecpu = $conf->{runningcpu};
4962         print "Resuming suspended VM\n";
4963     }
4964
4965     my ($cmd, $vollist, $spice_port) =
4966         config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
4967
4968     my $migration_ip;
4969     my $get_migration_ip = sub {
4970         my ($nodename) = @_;
4971
4972         return $migration_ip if defined($migration_ip);
4973
4974         my $cidr = $migrate_opts->{network};
4975
4976         if (!defined($cidr)) {
4977             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4978             $cidr = $dc_conf->{migration}->{network};
4979         }
4980
4981         if (defined($cidr)) {
4982             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
4983
4984             die "could not get IP: no address configured on local " .
4985                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
4986
4987             die "could not get IP: multiple addresses configured on local " .
4988                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
4989
4990             $migration_ip = @$ips[0];
4991         }
4992
4993         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
4994             if !defined($migration_ip);
4995
4996         return $migration_ip;
4997     };
4998
4999     my $migrate_uri;
5000     if ($statefile) {
5001         if ($statefile eq 'tcp') {
5002             my $localip = "localhost";
5003             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5004             my $nodename = nodename();
5005
5006             if (!defined($migration_type)) {
5007                 if (defined($datacenterconf->{migration}->{type})) {
5008                     $migration_type = $datacenterconf->{migration}->{type};
5009                 } else {
5010                     $migration_type = 'secure';
5011                 }
5012             }
5013
5014             if ($migration_type eq 'insecure') {
5015                 $localip = $get_migration_ip->($nodename);
5016                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5017             }
5018
5019             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5020             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5021             $migrate_uri = "tcp:${localip}:${migrate_port}";
5022             push @$cmd, '-incoming', $migrate_uri;
5023             push @$cmd, '-S';
5024
5025         } elsif ($statefile eq 'unix') {
5026             # should be default for secure migrations as a ssh TCP forward
5027             # tunnel is not deterministic reliable ready and fails regurarly
5028             # to set up in time, so use UNIX socket forwards
5029             my $socket_addr = "/run/qemu-server/$vmid.migrate";
5030             unlink $socket_addr;
5031
5032             $migrate_uri = "unix:$socket_addr";
5033
5034             push @$cmd, '-incoming', $migrate_uri;
5035             push @$cmd, '-S';
5036
5037         } elsif (-e $statefile) {
5038             push @$cmd, '-loadstate', $statefile;
5039         } else {
5040             my $statepath = PVE::Storage::path($storecfg, $statefile);
5041             push @$vollist, $statefile;
5042             push @$cmd, '-loadstate', $statepath;
5043         }
5044     } elsif ($params->{paused}) {
5045         push @$cmd, '-S';
5046     }
5047
5048     # host pci devices
5049     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5050       my $d = parse_hostpci($conf->{"hostpci$i"});
5051       next if !$d;
5052       my $pcidevices = $d->{pciid};
5053       foreach my $pcidevice (@$pcidevices) {
5054             my $pciid = $pcidevice->{id};
5055
5056             my $info = PVE::SysFSTools::pci_device_info("$pciid");
5057             die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5058             die "no pci device info for device '$pciid'\n" if !$info;
5059
5060             if ($d->{mdev}) {
5061                 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5062                 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5063             } else {
5064                 die "can't unbind/bind PCI group to VFIO '$pciid'\n"
5065                     if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5066                 die "can't reset PCI device '$pciid'\n"
5067                     if $info->{has_fl_reset} && !PVE::SysFSTools::pci_dev_reset($info);
5068             }
5069       }
5070     }
5071
5072     PVE::Storage::activate_volumes($storecfg, $vollist);
5073
5074     eval {
5075         run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5076             outfunc => sub {}, errfunc => sub {});
5077     };
5078     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5079     # timeout should be more than enough here...
5080     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5081
5082     my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5083                                               : $defaults->{cpuunits};
5084
5085     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5086     my %run_params = (
5087         timeout => $statefile ? undef : $start_timeout,
5088         umask => 0077,
5089         noerr => 1,
5090     );
5091
5092     # when migrating, prefix QEMU output so other side can pick up any
5093     # errors that might occur and show the user
5094     if ($migratedfrom) {
5095         $run_params{quiet} = 1;
5096         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5097     }
5098
5099     my %properties = (
5100         Slice => 'qemu.slice',
5101         KillMode => 'none'
5102     );
5103
5104     if (PVE::CGroup::cgroup_mode() == 2) {
5105         $properties{CPUWeight} = $cpuunits;
5106     } else {
5107         $properties{CPUShares} = $cpuunits;
5108     }
5109
5110     if (my $cpulimit = $conf->{cpulimit}) {
5111         $properties{CPUQuota} = int($cpulimit * 100);
5112     }
5113     $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5114
5115     my $run_qemu = sub {
5116         PVE::Tools::run_fork sub {
5117             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5118
5119             my $exitcode = run_command($cmd, %run_params);
5120             die "QEMU exited with code $exitcode\n" if $exitcode;
5121         };
5122     };
5123
5124     if ($conf->{hugepages}) {
5125
5126         my $code = sub {
5127             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5128             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5129
5130             PVE::QemuServer::Memory::hugepages_mount();
5131             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5132
5133             eval { $run_qemu->() };
5134             if (my $err = $@) {
5135                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5136                     if !$conf->{keephugepages};
5137                 die $err;
5138             }
5139
5140             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5141                 if !$conf->{keephugepages};
5142         };
5143         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5144
5145     } else {
5146         eval { $run_qemu->() };
5147     }
5148
5149     if (my $err = $@) {
5150         # deactivate volumes if start fails
5151         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5152         die "start failed: $err";
5153     }
5154
5155     print "migration listens on $migrate_uri\n" if $migrate_uri;
5156     $res->{migrate_uri} = $migrate_uri;
5157
5158     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5159         eval { mon_cmd($vmid, "cont"); };
5160         warn $@ if $@;
5161     }
5162
5163     #start nbd server for storage migration
5164     if (my $nbd = $migrate_opts->{nbd}) {
5165         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5166
5167         my $migrate_storage_uri;
5168         # nbd_protocol_version > 0 for unix socket support
5169         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5170             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5171             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5172             $migrate_storage_uri = "nbd:unix:$socket_path";
5173         } else {
5174             my $nodename = nodename();
5175             my $localip = $get_migration_ip->($nodename);
5176             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5177             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5178
5179             mon_cmd($vmid, "nbd-server-start", addr => {
5180                 type => 'inet',
5181                 data => {
5182                     host => "${localip}",
5183                     port => "${storage_migrate_port}",
5184                 },
5185             });
5186             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5187             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5188         }
5189
5190         $res->{migrate_storage_uri} = $migrate_storage_uri;
5191
5192         foreach my $opt (sort keys %$nbd) {
5193             my $drivestr = $nbd->{$opt}->{drivestr};
5194             my $volid = $nbd->{$opt}->{volid};
5195             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5196             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5197             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5198             print "re-using replicated volume: $opt - $volid\n"
5199                 if $nbd->{$opt}->{replicated};
5200
5201             $res->{drives}->{$opt} = $nbd->{$opt};
5202             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5203         }
5204     }
5205
5206     if ($migratedfrom) {
5207         eval {
5208             set_migration_caps($vmid);
5209         };
5210         warn $@ if $@;
5211
5212         if ($spice_port) {
5213             print "spice listens on port $spice_port\n";
5214             $res->{spice_port} = $spice_port;
5215             if ($migrate_opts->{spice_ticket}) {
5216                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5217                     $migrate_opts->{spice_ticket});
5218                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5219             }
5220         }
5221
5222     } else {
5223         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5224             if !$statefile && $conf->{balloon};
5225
5226         foreach my $opt (keys %$conf) {
5227             next if $opt !~  m/^net\d+$/;
5228             my $nicconf = parse_net($conf->{$opt});
5229             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5230         }
5231     }
5232
5233     mon_cmd($vmid, 'qom-set',
5234                 path => "machine/peripheral/balloon0",
5235                 property => "guest-stats-polling-interval",
5236                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5237
5238     if ($resume) {
5239         print "Resumed VM, removing state\n";
5240         if (my $vmstate = $conf->{vmstate}) {
5241             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5242             PVE::Storage::vdisk_free($storecfg, $vmstate);
5243         }
5244         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5245         PVE::QemuConfig->write_config($vmid, $conf);
5246     }
5247
5248     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5249
5250     return $res;
5251 }
5252
5253 sub vm_commandline {
5254     my ($storecfg, $vmid, $snapname) = @_;
5255
5256     my $conf = PVE::QemuConfig->load_config($vmid);
5257     my $forcemachine;
5258     my $forcecpu;
5259
5260     if ($snapname) {
5261         my $snapshot = $conf->{snapshots}->{$snapname};
5262         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5263
5264         # check for machine or CPU overrides in snapshot
5265         $forcemachine = $snapshot->{runningmachine};
5266         $forcecpu = $snapshot->{runningcpu};
5267
5268         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5269
5270         $conf = $snapshot;
5271     }
5272
5273     my $defaults = load_defaults();
5274
5275     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults,
5276         $forcemachine, $forcecpu);
5277
5278     return PVE::Tools::cmd2string($cmd);
5279 }
5280
5281 sub vm_reset {
5282     my ($vmid, $skiplock) = @_;
5283
5284     PVE::QemuConfig->lock_config($vmid, sub {
5285
5286         my $conf = PVE::QemuConfig->load_config($vmid);
5287
5288         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5289
5290         mon_cmd($vmid, "system_reset");
5291     });
5292 }
5293
5294 sub get_vm_volumes {
5295     my ($conf) = @_;
5296
5297     my $vollist = [];
5298     foreach_volid($conf, sub {
5299         my ($volid, $attr) = @_;
5300
5301         return if $volid =~ m|^/|;
5302
5303         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5304         return if !$sid;
5305
5306         push @$vollist, $volid;
5307     });
5308
5309     return $vollist;
5310 }
5311
5312 sub vm_stop_cleanup {
5313     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5314
5315     eval {
5316
5317         if (!$keepActive) {
5318             my $vollist = get_vm_volumes($conf);
5319             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5320         }
5321
5322         foreach my $ext (qw(mon qmp pid vnc qga)) {
5323             unlink "/var/run/qemu-server/${vmid}.$ext";
5324         }
5325
5326         if ($conf->{ivshmem}) {
5327             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5328             # just delete it for now, VMs which have this already open do not
5329             # are affected, but new VMs will get a separated one. If this
5330             # becomes an issue we either add some sort of ref-counting or just
5331             # add a "don't delete on stop" flag to the ivshmem format.
5332             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5333         }
5334
5335         foreach my $key (keys %$conf) {
5336             next if $key !~ m/^hostpci(\d+)$/;
5337             my $hostpciindex = $1;
5338             my $d = parse_hostpci($conf->{$key});
5339             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5340
5341             foreach my $pci (@{$d->{pciid}}) {
5342                 my $pciid = $pci->{id};
5343                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5344             }
5345         }
5346
5347         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5348     };
5349     warn $@ if $@; # avoid errors - just warn
5350 }
5351
5352 # call only in locked context
5353 sub _do_vm_stop {
5354     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5355
5356     my $pid = check_running($vmid, $nocheck);
5357     return if !$pid;
5358
5359     my $conf;
5360     if (!$nocheck) {
5361         $conf = PVE::QemuConfig->load_config($vmid);
5362         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5363         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5364             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5365             $timeout = $opts->{down} if $opts->{down};
5366         }
5367         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5368     }
5369
5370     eval {
5371         if ($shutdown) {
5372             if (defined($conf) && get_qga_key($conf, 'enabled')) {
5373                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5374             } else {
5375                 mon_cmd($vmid, "system_powerdown");
5376             }
5377         } else {
5378             mon_cmd($vmid, "quit");
5379         }
5380     };
5381     my $err = $@;
5382
5383     if (!$err) {
5384         $timeout = 60 if !defined($timeout);
5385
5386         my $count = 0;
5387         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5388             $count++;
5389             sleep 1;
5390         }
5391
5392         if ($count >= $timeout) {
5393             if ($force) {
5394                 warn "VM still running - terminating now with SIGTERM\n";
5395                 kill 15, $pid;
5396             } else {
5397                 die "VM quit/powerdown failed - got timeout\n";
5398             }
5399         } else {
5400             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5401             return;
5402         }
5403     } else {
5404        if (!check_running($vmid, $nocheck)) {
5405             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5406             return;
5407        }
5408        if ($force) {
5409             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5410             kill 15, $pid;
5411         } else {
5412             die "VM quit/powerdown failed\n";
5413         }
5414     }
5415
5416     # wait again
5417     $timeout = 10;
5418
5419     my $count = 0;
5420     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5421         $count++;
5422         sleep 1;
5423     }
5424
5425     if ($count >= $timeout) {
5426         warn "VM still running - terminating now with SIGKILL\n";
5427         kill 9, $pid;
5428         sleep 1;
5429     }
5430
5431     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5432 }
5433
5434 # Note: use $nocheck to skip tests if VM configuration file exists.
5435 # We need that when migration VMs to other nodes (files already moved)
5436 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5437 sub vm_stop {
5438     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5439
5440     $force = 1 if !defined($force) && !$shutdown;
5441
5442     if ($migratedfrom){
5443         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5444         kill 15, $pid if $pid;
5445         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5446         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5447         return;
5448     }
5449
5450     PVE::QemuConfig->lock_config($vmid, sub {
5451         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5452    });
5453 }
5454
5455 sub vm_reboot {
5456     my ($vmid, $timeout) = @_;
5457
5458     PVE::QemuConfig->lock_config($vmid, sub {
5459         eval {
5460
5461             # only reboot if running, as qmeventd starts it again on a stop event
5462             return if !check_running($vmid);
5463
5464             create_reboot_request($vmid);
5465
5466             my $storecfg = PVE::Storage::config();
5467             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5468
5469         };
5470         if (my $err = $@) {
5471             # avoid that the next normal shutdown will be confused for a reboot
5472             clear_reboot_request($vmid);
5473             die $err;
5474         }
5475    });
5476 }
5477
5478 # note: if using the statestorage parameter, the caller has to check privileges
5479 sub vm_suspend {
5480     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5481
5482     my $conf;
5483     my $path;
5484     my $storecfg;
5485     my $vmstate;
5486
5487     PVE::QemuConfig->lock_config($vmid, sub {
5488
5489         $conf = PVE::QemuConfig->load_config($vmid);
5490
5491         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5492         PVE::QemuConfig->check_lock($conf)
5493             if !($skiplock || $is_backing_up);
5494
5495         die "cannot suspend to disk during backup\n"
5496             if $is_backing_up && $includestate;
5497
5498         if ($includestate) {
5499             $conf->{lock} = 'suspending';
5500             my $date = strftime("%Y-%m-%d", localtime(time()));
5501             $storecfg = PVE::Storage::config();
5502             if (!$statestorage) {
5503                 $statestorage = find_vmstate_storage($conf, $storecfg);
5504                 # check permissions for the storage
5505                 my $rpcenv = PVE::RPCEnvironment::get();
5506                 if ($rpcenv->{type} ne 'cli') {
5507                     my $authuser = $rpcenv->get_user();
5508                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5509                 }
5510             }
5511
5512
5513             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
5514                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5515             $path = PVE::Storage::path($storecfg, $vmstate);
5516             PVE::QemuConfig->write_config($vmid, $conf);
5517         } else {
5518             mon_cmd($vmid, "stop");
5519         }
5520     });
5521
5522     if ($includestate) {
5523         # save vm state
5524         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5525
5526         eval {
5527             mon_cmd($vmid, "savevm-start", statefile => $path);
5528             for(;;) {
5529                 my $state = mon_cmd($vmid, "query-savevm");
5530                 if (!$state->{status}) {
5531                     die "savevm not active\n";
5532                 } elsif ($state->{status} eq 'active') {
5533                     sleep(1);
5534                     next;
5535                 } elsif ($state->{status} eq 'completed') {
5536                     print "State saved, quitting\n";
5537                     last;
5538                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5539                     die "query-savevm failed with error '$state->{error}'\n"
5540                 } else {
5541                     die "query-savevm returned status '$state->{status}'\n";
5542                 }
5543             }
5544         };
5545         my $err = $@;
5546
5547         PVE::QemuConfig->lock_config($vmid, sub {
5548             $conf = PVE::QemuConfig->load_config($vmid);
5549             if ($err) {
5550                 # cleanup, but leave suspending lock, to indicate something went wrong
5551                 eval {
5552                     mon_cmd($vmid, "savevm-end");
5553                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5554                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5555                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
5556                     PVE::QemuConfig->write_config($vmid, $conf);
5557                 };
5558                 warn $@ if $@;
5559                 die $err;
5560             }
5561
5562             die "lock changed unexpectedly\n"
5563                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5564
5565             mon_cmd($vmid, "quit");
5566             $conf->{lock} = 'suspended';
5567             PVE::QemuConfig->write_config($vmid, $conf);
5568         });
5569     }
5570 }
5571
5572 sub vm_resume {
5573     my ($vmid, $skiplock, $nocheck) = @_;
5574
5575     PVE::QemuConfig->lock_config($vmid, sub {
5576         my $res = mon_cmd($vmid, 'query-status');
5577         my $resume_cmd = 'cont';
5578         my $reset = 0;
5579
5580         if ($res->{status}) {
5581             return if $res->{status} eq 'running'; # job done, go home
5582             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
5583             $reset = 1 if $res->{status} eq 'shutdown';
5584         }
5585
5586         if (!$nocheck) {
5587
5588             my $conf = PVE::QemuConfig->load_config($vmid);
5589
5590             PVE::QemuConfig->check_lock($conf)
5591                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5592         }
5593
5594         if ($reset) {
5595             # required if a VM shuts down during a backup and we get a resume
5596             # request before the backup finishes for example
5597             mon_cmd($vmid, "system_reset");
5598         }
5599         mon_cmd($vmid, $resume_cmd);
5600     });
5601 }
5602
5603 sub vm_sendkey {
5604     my ($vmid, $skiplock, $key) = @_;
5605
5606     PVE::QemuConfig->lock_config($vmid, sub {
5607
5608         my $conf = PVE::QemuConfig->load_config($vmid);
5609
5610         # there is no qmp command, so we use the human monitor command
5611         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5612         die $res if $res ne '';
5613     });
5614 }
5615
5616 # vzdump restore implementaion
5617
5618 sub tar_archive_read_firstfile {
5619     my $archive = shift;
5620
5621     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5622
5623     # try to detect archive type first
5624     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5625         die "unable to open file '$archive'\n";
5626     my $firstfile = <$fh>;
5627     kill 15, $pid;
5628     close $fh;
5629
5630     die "ERROR: archive contaions no data\n" if !$firstfile;
5631     chomp $firstfile;
5632
5633     return $firstfile;
5634 }
5635
5636 sub tar_restore_cleanup {
5637     my ($storecfg, $statfile) = @_;
5638
5639     print STDERR "starting cleanup\n";
5640
5641     if (my $fd = IO::File->new($statfile, "r")) {
5642         while (defined(my $line = <$fd>)) {
5643             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5644                 my $volid = $2;
5645                 eval {
5646                     if ($volid =~ m|^/|) {
5647                         unlink $volid || die 'unlink failed\n';
5648                     } else {
5649                         PVE::Storage::vdisk_free($storecfg, $volid);
5650                     }
5651                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5652                 };
5653                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5654             } else {
5655                 print STDERR "unable to parse line in statfile - $line";
5656             }
5657         }
5658         $fd->close();
5659     }
5660 }
5661
5662 sub restore_file_archive {
5663     my ($archive, $vmid, $user, $opts) = @_;
5664
5665     return restore_vma_archive($archive, $vmid, $user, $opts)
5666         if $archive eq '-';
5667
5668     my $info = PVE::Storage::archive_info($archive);
5669     my $format = $opts->{format} // $info->{format};
5670     my $comp = $info->{compression};
5671
5672     # try to detect archive format
5673     if ($format eq 'tar') {
5674         return restore_tar_archive($archive, $vmid, $user, $opts);
5675     } else {
5676         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5677     }
5678 }
5679
5680 # hepler to remove disks that will not be used after restore
5681 my $restore_cleanup_oldconf = sub {
5682     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
5683
5684     PVE::QemuConfig->foreach_volume($oldconf, sub {
5685         my ($ds, $drive) = @_;
5686
5687         return if drive_is_cdrom($drive, 1);
5688
5689         my $volid = $drive->{file};
5690         return if !$volid || $volid =~ m|^/|;
5691
5692         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
5693         return if !$path || !$owner || ($owner != $vmid);
5694
5695         # Note: only delete disk we want to restore
5696         # other volumes will become unused
5697         if ($virtdev_hash->{$ds}) {
5698             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
5699             if (my $err = $@) {
5700                 warn $err;
5701             }
5702         }
5703     });
5704
5705     # delete vmstate files, after the restore we have no snapshots anymore
5706     foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5707         my $snap = $oldconf->{snapshots}->{$snapname};
5708         if ($snap->{vmstate}) {
5709             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
5710             if (my $err = $@) {
5711                 warn $err;
5712             }
5713         }
5714     }
5715 };
5716
5717 # Helper to parse vzdump backup device hints
5718 #
5719 # $rpcenv: Environment, used to ckeck storage permissions
5720 # $user: User ID, to check storage permissions
5721 # $storecfg: Storage configuration
5722 # $fh: the file handle for reading the configuration
5723 # $devinfo: should contain device sizes for all backu-up'ed devices
5724 # $options: backup options (pool, default storage)
5725 #
5726 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
5727 my $parse_backup_hints = sub {
5728     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
5729
5730     my $virtdev_hash = {};
5731
5732     while (defined(my $line = <$fh>)) {
5733         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5734             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5735             die "archive does not contain data for drive '$virtdev'\n"
5736                 if !$devinfo->{$devname};
5737
5738             if (defined($options->{storage})) {
5739                 $storeid = $options->{storage} || 'local';
5740             } elsif (!$storeid) {
5741                 $storeid = 'local';
5742             }
5743             $format = 'raw' if !$format;
5744             $devinfo->{$devname}->{devname} = $devname;
5745             $devinfo->{$devname}->{virtdev} = $virtdev;
5746             $devinfo->{$devname}->{format} = $format;
5747             $devinfo->{$devname}->{storeid} = $storeid;
5748
5749             # check permission on storage
5750             my $pool = $options->{pool}; # todo: do we need that?
5751             if ($user ne 'root@pam') {
5752                 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5753             }
5754
5755             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5756         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
5757             my $virtdev = $1;
5758             my $drive = parse_drive($virtdev, $2);
5759             if (drive_is_cloudinit($drive)) {
5760                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5761                 $storeid = $options->{storage} if defined ($options->{storage});
5762                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5763                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
5764
5765                 $virtdev_hash->{$virtdev} = {
5766                     format => $format,
5767                     storeid => $storeid,
5768                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
5769                     is_cloudinit => 1,
5770                 };
5771             }
5772         }
5773     }
5774
5775     return $virtdev_hash;
5776 };
5777
5778 # Helper to allocate and activate all volumes required for a restore
5779 #
5780 # $storecfg: Storage configuration
5781 # $virtdev_hash: as returned by parse_backup_hints()
5782 #
5783 # Returns: { $virtdev => $volid }
5784 my $restore_allocate_devices = sub {
5785     my ($storecfg, $virtdev_hash, $vmid) = @_;
5786
5787     my $map = {};
5788     foreach my $virtdev (sort keys %$virtdev_hash) {
5789         my $d = $virtdev_hash->{$virtdev};
5790         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5791         my $storeid = $d->{storeid};
5792         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5793
5794         # test if requested format is supported
5795         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5796         my $supported = grep { $_ eq $d->{format} } @$validFormats;
5797         $d->{format} = $defFormat if !$supported;
5798
5799         my $name;
5800         if ($d->{is_cloudinit}) {
5801             $name = "vm-$vmid-cloudinit";
5802             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5803             if ($scfg->{path}) {
5804                 $name .= ".$d->{format}";
5805             }
5806         }
5807
5808         my $volid = PVE::Storage::vdisk_alloc(
5809             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
5810
5811         print STDERR "new volume ID is '$volid'\n";
5812         $d->{volid} = $volid;
5813
5814         PVE::Storage::activate_volumes($storecfg, [$volid]);
5815
5816         $map->{$virtdev} = $volid;
5817     }
5818
5819     return $map;
5820 };
5821
5822 my $restore_update_config_line = sub {
5823     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5824
5825     return if $line =~ m/^\#qmdump\#/;
5826     return if $line =~ m/^\#vzdump\#/;
5827     return if $line =~ m/^lock:/;
5828     return if $line =~ m/^unused\d+:/;
5829     return if $line =~ m/^parent:/;
5830
5831     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5832     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5833         # try to convert old 1.X settings
5834         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5835         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5836             my ($model, $macaddr) = split(/\=/, $devconfig);
5837             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5838             my $net = {
5839                 model => $model,
5840                 bridge => "vmbr$ind",
5841                 macaddr => $macaddr,
5842             };
5843             my $netstr = print_net($net);
5844
5845             print $outfd "net$cookie->{netcount}: $netstr\n";
5846             $cookie->{netcount}++;
5847         }
5848     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5849         my ($id, $netstr) = ($1, $2);
5850         my $net = parse_net($netstr);
5851         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5852         $netstr = print_net($net);
5853         print $outfd "$id: $netstr\n";
5854     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5855         my $virtdev = $1;
5856         my $value = $3;
5857         my $di = parse_drive($virtdev, $value);
5858         if (defined($di->{backup}) && !$di->{backup}) {
5859             print $outfd "#$line";
5860         } elsif ($map->{$virtdev}) {
5861             delete $di->{format}; # format can change on restore
5862             $di->{file} = $map->{$virtdev};
5863             $value = print_drive($di);
5864             print $outfd "$virtdev: $value\n";
5865         } else {
5866             print $outfd $line;
5867         }
5868     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5869         my $vmgenid = $1;
5870         if ($vmgenid ne '0') {
5871             # always generate a new vmgenid if there was a valid one setup
5872             $vmgenid = generate_uuid();
5873         }
5874         print $outfd "vmgenid: $vmgenid\n";
5875     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5876         my ($uuid, $uuid_str);
5877         UUID::generate($uuid);
5878         UUID::unparse($uuid, $uuid_str);
5879         my $smbios1 = parse_smbios1($2);
5880         $smbios1->{uuid} = $uuid_str;
5881         print $outfd $1.print_smbios1($smbios1)."\n";
5882     } else {
5883         print $outfd $line;
5884     }
5885 };
5886
5887 my $restore_deactivate_volumes = sub {
5888     my ($storecfg, $devinfo) = @_;
5889
5890     my $vollist = [];
5891     foreach my $devname (keys %$devinfo) {
5892         my $volid = $devinfo->{$devname}->{volid};
5893         push @$vollist, $volid if $volid;
5894     }
5895
5896     PVE::Storage::deactivate_volumes($storecfg, $vollist);
5897 };
5898
5899 my $restore_destroy_volumes = sub {
5900     my ($storecfg, $devinfo) = @_;
5901
5902     foreach my $devname (keys %$devinfo) {
5903         my $volid = $devinfo->{$devname}->{volid};
5904         next if !$volid;
5905         eval {
5906             if ($volid =~ m|^/|) {
5907                 unlink $volid || die 'unlink failed\n';
5908             } else {
5909                 PVE::Storage::vdisk_free($storecfg, $volid);
5910             }
5911             print STDERR "temporary volume '$volid' sucessfuly removed\n";
5912         };
5913         print STDERR "unable to cleanup '$volid' - $@" if $@;
5914     }
5915 };
5916
5917 sub scan_volids {
5918     my ($cfg, $vmid) = @_;
5919
5920     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5921
5922     my $volid_hash = {};
5923     foreach my $storeid (keys %$info) {
5924         foreach my $item (@{$info->{$storeid}}) {
5925             next if !($item->{volid} && $item->{size});
5926             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5927             $volid_hash->{$item->{volid}} = $item;
5928         }
5929     }
5930
5931     return $volid_hash;
5932 }
5933
5934 sub update_disk_config {
5935     my ($vmid, $conf, $volid_hash) = @_;
5936
5937     my $changes;
5938     my $prefix = "VM $vmid";
5939
5940     # used and unused disks
5941     my $referenced = {};
5942
5943     # Note: it is allowed to define multiple storages with same path (alias), so
5944     # we need to check both 'volid' and real 'path' (two different volid can point
5945     # to the same path).
5946
5947     my $referencedpath = {};
5948
5949     # update size info
5950     PVE::QemuConfig->foreach_volume($conf, sub {
5951         my ($opt, $drive) = @_;
5952
5953         my $volid = $drive->{file};
5954         return if !$volid;
5955         my $volume = $volid_hash->{$volid};
5956
5957         # mark volid as "in-use" for next step
5958         $referenced->{$volid} = 1;
5959         if ($volume && (my $path = $volume->{path})) {
5960             $referencedpath->{$path} = 1;
5961         }
5962
5963         return if drive_is_cdrom($drive);
5964         return if !$volume;
5965
5966         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
5967         if (defined($updated)) {
5968             $changes = 1;
5969             $conf->{$opt} = print_drive($updated);
5970             print "$prefix ($opt): $msg\n";
5971         }
5972     });
5973
5974     # remove 'unusedX' entry if volume is used
5975     PVE::QemuConfig->foreach_unused_volume($conf, sub {
5976         my ($opt, $drive) = @_;
5977
5978         my $volid = $drive->{file};
5979         return if !$volid;
5980
5981         my $path;
5982         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5983         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5984             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
5985             $changes = 1;
5986             delete $conf->{$opt};
5987         }
5988
5989         $referenced->{$volid} = 1;
5990         $referencedpath->{$path} = 1 if $path;
5991     });
5992
5993     foreach my $volid (sort keys %$volid_hash) {
5994         next if $volid =~ m/vm-$vmid-state-/;
5995         next if $referenced->{$volid};
5996         my $path = $volid_hash->{$volid}->{path};
5997         next if !$path; # just to be sure
5998         next if $referencedpath->{$path};
5999         $changes = 1;
6000         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6001         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6002         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6003     }
6004
6005     return $changes;
6006 }
6007
6008 sub rescan {
6009     my ($vmid, $nolock, $dryrun) = @_;
6010
6011     my $cfg = PVE::Storage::config();
6012
6013     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6014     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6015     foreach my $stor (keys %{$cfg->{ids}}) {
6016         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6017     }
6018
6019     print "rescan volumes...\n";
6020     my $volid_hash = scan_volids($cfg, $vmid);
6021
6022     my $updatefn =  sub {
6023         my ($vmid) = @_;
6024
6025         my $conf = PVE::QemuConfig->load_config($vmid);
6026
6027         PVE::QemuConfig->check_lock($conf);
6028
6029         my $vm_volids = {};
6030         foreach my $volid (keys %$volid_hash) {
6031             my $info = $volid_hash->{$volid};
6032             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6033         }
6034
6035         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6036
6037         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6038     };
6039
6040     if (defined($vmid)) {
6041         if ($nolock) {
6042             &$updatefn($vmid);
6043         } else {
6044             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6045         }
6046     } else {
6047         my $vmlist = config_list();
6048         foreach my $vmid (keys %$vmlist) {
6049             if ($nolock) {
6050                 &$updatefn($vmid);
6051             } else {
6052                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6053             }
6054         }
6055     }
6056 }
6057
6058 sub restore_proxmox_backup_archive {
6059     my ($archive, $vmid, $user, $options) = @_;
6060
6061     my $storecfg = PVE::Storage::config();
6062
6063     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6064     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6065
6066     my $fingerprint = $scfg->{fingerprint};
6067     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6068
6069     my $repo = PVE::PBSClient::get_repository($scfg);
6070
6071     # This is only used for `pbs-restore`!
6072     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6073     local $ENV{PBS_PASSWORD} = $password;
6074     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6075
6076     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6077         PVE::Storage::parse_volname($storecfg, $archive);
6078
6079     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6080
6081     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6082
6083     my $tmpdir = "/var/tmp/vzdumptmp$$";
6084     rmtree $tmpdir;
6085     mkpath $tmpdir;
6086
6087     my $conffile = PVE::QemuConfig->config_file($vmid);
6088     my $tmpfn = "$conffile.$$.tmp";
6089      # disable interrupts (always do cleanups)
6090     local $SIG{INT} =
6091         local $SIG{TERM} =
6092         local $SIG{QUIT} =
6093         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6094
6095     # Note: $oldconf is undef if VM does not exists
6096     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6097     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6098
6099     my $rpcenv = PVE::RPCEnvironment::get();
6100     my $devinfo = {};
6101
6102     eval {
6103         # enable interrupts
6104         local $SIG{INT} =
6105             local $SIG{TERM} =
6106             local $SIG{QUIT} =
6107             local $SIG{HUP} =
6108             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6109
6110         my $cfgfn = "$tmpdir/qemu-server.conf";
6111         my $firewall_config_fn = "$tmpdir/fw.conf";
6112         my $index_fn = "$tmpdir/index.json";
6113
6114         my $cmd = "restore";
6115
6116         my $param = [$pbs_backup_name, "index.json", $index_fn];
6117         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6118         my $index = PVE::Tools::file_get_contents($index_fn);
6119         $index = decode_json($index);
6120
6121         # print Dumper($index);
6122         foreach my $info (@{$index->{files}}) {
6123             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6124                 my $devname = $1;
6125                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6126                     $devinfo->{$devname}->{size} = $1;
6127                 } else {
6128                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6129                 }
6130             }
6131         }
6132
6133         my $is_qemu_server_backup = scalar(
6134             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6135         );
6136         if (!$is_qemu_server_backup) {
6137             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6138         }
6139         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6140
6141         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6142         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6143
6144         if ($has_firewall_config) {
6145             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6146             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6147
6148             my $pve_firewall_dir = '/etc/pve/firewall';
6149             mkdir $pve_firewall_dir; # make sure the dir exists
6150             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6151         }
6152
6153         my $fh = IO::File->new($cfgfn, "r") ||
6154             die "unable to read qemu-server.conf - $!\n";
6155
6156         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6157
6158         # fixme: rate limit?
6159
6160         # create empty/temp config
6161         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6162
6163         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6164
6165         # allocate volumes
6166         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6167
6168         foreach my $virtdev (sort keys %$virtdev_hash) {
6169             my $d = $virtdev_hash->{$virtdev};
6170             next if $d->{is_cloudinit}; # no need to restore cloudinit
6171
6172             my $volid = $d->{volid};
6173
6174             my $path = PVE::Storage::path($storecfg, $volid);
6175
6176             # This is the ONLY user of the PBS_ env vars set on top of this function!
6177             my $pbs_restore_cmd = [
6178                 '/usr/bin/pbs-restore',
6179                 '--repository', $repo,
6180                 $pbs_backup_name,
6181                 "$d->{devname}.img.fidx",
6182                 $path,
6183                 '--verbose',
6184                 ];
6185
6186             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6187             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
6188
6189             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6190                 push @$pbs_restore_cmd, '--skip-zero';
6191             }
6192
6193             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6194             print "restore proxmox backup image: $dbg_cmdstring\n";
6195             run_command($pbs_restore_cmd);
6196         }
6197
6198         $fh->seek(0, 0) || die "seek failed - $!\n";
6199
6200         my $outfd = IO::File->new($tmpfn, "w") || die "unable to write config for VM $vmid\n";
6201
6202         my $cookie = { netcount => 0 };
6203         while (defined(my $line = <$fh>)) {
6204             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $options->{unique});
6205         }
6206
6207         $fh->close();
6208         $outfd->close();
6209     };
6210     my $err = $@;
6211
6212     $restore_deactivate_volumes->($storecfg, $devinfo);
6213
6214     rmtree $tmpdir;
6215
6216     if ($err) {
6217         unlink $tmpfn;
6218         $restore_destroy_volumes->($storecfg, $devinfo);
6219         die $err;
6220     }
6221
6222     rename($tmpfn, $conffile) ||
6223         die "unable to commit configuration file '$conffile'\n";
6224
6225     PVE::Cluster::cfs_update(); # make sure we read new file
6226
6227     eval { rescan($vmid, 1); };
6228     warn $@ if $@;
6229 }
6230
6231 sub restore_vma_archive {
6232     my ($archive, $vmid, $user, $opts, $comp) = @_;
6233
6234     my $readfrom = $archive;
6235
6236     my $cfg = PVE::Storage::config();
6237     my $commands = [];
6238     my $bwlimit = $opts->{bwlimit};
6239
6240     my $dbg_cmdstring = '';
6241     my $add_pipe = sub {
6242         my ($cmd) = @_;
6243         push @$commands, $cmd;
6244         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6245         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6246         $readfrom = '-';
6247     };
6248
6249     my $input = undef;
6250     if ($archive eq '-') {
6251         $input = '<&STDIN';
6252     } else {
6253         # If we use a backup from a PVE defined storage we also consider that
6254         # storage's rate limit:
6255         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6256         if (defined($volid)) {
6257             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6258             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6259             if ($readlimit) {
6260                 print STDERR "applying read rate limit: $readlimit\n";
6261                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6262                 $add_pipe->($cstream);
6263             }
6264         }
6265     }
6266
6267     if ($comp) {
6268         my $info = PVE::Storage::decompressor_info('vma', $comp);
6269         my $cmd = $info->{decompressor};
6270         push @$cmd, $readfrom;
6271         $add_pipe->($cmd);
6272     }
6273
6274     my $tmpdir = "/var/tmp/vzdumptmp$$";
6275     rmtree $tmpdir;
6276
6277     # disable interrupts (always do cleanups)
6278     local $SIG{INT} =
6279         local $SIG{TERM} =
6280         local $SIG{QUIT} =
6281         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6282
6283     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6284     POSIX::mkfifo($mapfifo, 0600);
6285     my $fifofh;
6286     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
6287
6288     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6289
6290     my $oldtimeout;
6291     my $timeout = 5;
6292
6293     my $devinfo = {};
6294
6295     my $rpcenv = PVE::RPCEnvironment::get();
6296
6297     my $conffile = PVE::QemuConfig->config_file($vmid);
6298     my $tmpfn = "$conffile.$$.tmp";
6299
6300     # Note: $oldconf is undef if VM does not exist
6301     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6302     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6303
6304     my %storage_limits;
6305
6306     my $print_devmap = sub {
6307         my $cfgfn = "$tmpdir/qemu-server.conf";
6308
6309         # we can read the config - that is already extracted
6310         my $fh = IO::File->new($cfgfn, "r") ||
6311             die "unable to read qemu-server.conf - $!\n";
6312
6313         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6314         if (-f $fwcfgfn) {
6315             my $pve_firewall_dir = '/etc/pve/firewall';
6316             mkdir $pve_firewall_dir; # make sure the dir exists
6317             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6318         }
6319
6320         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6321
6322         foreach my $key (keys %storage_limits) {
6323             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6324             next if !$limit;
6325             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6326             $storage_limits{$key} = $limit * 1024;
6327         }
6328
6329         foreach my $devname (keys %$devinfo) {
6330             die "found no device mapping information for device '$devname'\n"
6331                 if !$devinfo->{$devname}->{virtdev};
6332         }
6333
6334         # create empty/temp config
6335         if ($oldconf) {
6336             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6337             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6338         }
6339
6340         # allocate volumes
6341         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6342
6343         # print restore information to $fifofh
6344         foreach my $virtdev (sort keys %$virtdev_hash) {
6345             my $d = $virtdev_hash->{$virtdev};
6346             next if $d->{is_cloudinit}; # no need to restore cloudinit
6347
6348             my $storeid = $d->{storeid};
6349             my $volid = $d->{volid};
6350
6351             my $map_opts = '';
6352             if (my $limit = $storage_limits{$storeid}) {
6353                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6354             }
6355
6356             my $write_zeros = 1;
6357             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6358                 $write_zeros = 0;
6359             }
6360
6361             my $path = PVE::Storage::path($cfg, $volid);
6362
6363             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6364
6365             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6366         }
6367
6368         $fh->seek(0, 0) || die "seek failed - $!\n";
6369
6370         my $outfd = IO::File->new($tmpfn, "w") || die "unable to write config for VM $vmid\n";
6371
6372         my $cookie = { netcount => 0 };
6373         while (defined(my $line = <$fh>)) {
6374             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6375         }
6376
6377         $fh->close();
6378         $outfd->close();
6379     };
6380
6381     eval {
6382         # enable interrupts
6383         local $SIG{INT} =
6384             local $SIG{TERM} =
6385             local $SIG{QUIT} =
6386             local $SIG{HUP} =
6387             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6388         local $SIG{ALRM} = sub { die "got timeout\n"; };
6389
6390         $oldtimeout = alarm($timeout);
6391
6392         my $parser = sub {
6393             my $line = shift;
6394
6395             print "$line\n";
6396
6397             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6398                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6399                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6400             } elsif ($line =~ m/^CTIME: /) {
6401                 # we correctly received the vma config, so we can disable
6402                 # the timeout now for disk allocation (set to 10 minutes, so
6403                 # that we always timeout if something goes wrong)
6404                 alarm(600);
6405                 &$print_devmap();
6406                 print $fifofh "done\n";
6407                 my $tmp = $oldtimeout || 0;
6408                 $oldtimeout = undef;
6409                 alarm($tmp);
6410                 close($fifofh);
6411                 $fifofh = undef;
6412             }
6413         };
6414
6415         print "restore vma archive: $dbg_cmdstring\n";
6416         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6417     };
6418     my $err = $@;
6419
6420     alarm($oldtimeout) if $oldtimeout;
6421
6422     $restore_deactivate_volumes->($cfg, $devinfo);
6423
6424     close($fifofh) if $fifofh;
6425     unlink $mapfifo;
6426     rmtree $tmpdir;
6427
6428     if ($err) {
6429         unlink $tmpfn;
6430         $restore_destroy_volumes->($cfg, $devinfo);
6431         die $err;
6432     }
6433
6434     rename($tmpfn, $conffile) ||
6435         die "unable to commit configuration file '$conffile'\n";
6436
6437     PVE::Cluster::cfs_update(); # make sure we read new file
6438
6439     eval { rescan($vmid, 1); };
6440     warn $@ if $@;
6441 }
6442
6443 sub restore_tar_archive {
6444     my ($archive, $vmid, $user, $opts) = @_;
6445
6446     if ($archive ne '-') {
6447         my $firstfile = tar_archive_read_firstfile($archive);
6448         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
6449             if $firstfile ne 'qemu-server.conf';
6450     }
6451
6452     my $storecfg = PVE::Storage::config();
6453
6454     # avoid zombie disks when restoring over an existing VM -> cleanup first
6455     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6456     # skiplock=1 because qmrestore has set the 'create' lock itself already
6457     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6458     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6459
6460     my $tocmd = "/usr/lib/qemu-server/qmextract";
6461
6462     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6463     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6464     $tocmd .= ' --prealloc' if $opts->{prealloc};
6465     $tocmd .= ' --info' if $opts->{info};
6466
6467     # tar option "xf" does not autodetect compression when read from STDIN,
6468     # so we pipe to zcat
6469     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6470         PVE::Tools::shellquote("--to-command=$tocmd");
6471
6472     my $tmpdir = "/var/tmp/vzdumptmp$$";
6473     mkpath $tmpdir;
6474
6475     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6476     local $ENV{VZDUMP_VMID} = $vmid;
6477     local $ENV{VZDUMP_USER} = $user;
6478
6479     my $conffile = PVE::QemuConfig->config_file($vmid);
6480     my $tmpfn = "$conffile.$$.tmp";
6481
6482     # disable interrupts (always do cleanups)
6483     local $SIG{INT} =
6484         local $SIG{TERM} =
6485         local $SIG{QUIT} =
6486         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6487
6488     eval {
6489         # enable interrupts
6490         local $SIG{INT} =
6491             local $SIG{TERM} =
6492             local $SIG{QUIT} =
6493             local $SIG{HUP} =
6494             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6495
6496         if ($archive eq '-') {
6497             print "extracting archive from STDIN\n";
6498             run_command($cmd, input => "<&STDIN");
6499         } else {
6500             print "extracting archive '$archive'\n";
6501             run_command($cmd);
6502         }
6503
6504         return if $opts->{info};
6505
6506         # read new mapping
6507         my $map = {};
6508         my $statfile = "$tmpdir/qmrestore.stat";
6509         if (my $fd = IO::File->new($statfile, "r")) {
6510             while (defined (my $line = <$fd>)) {
6511                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6512                     $map->{$1} = $2 if $1;
6513                 } else {
6514                     print STDERR "unable to parse line in statfile - $line\n";
6515                 }
6516             }
6517             $fd->close();
6518         }
6519
6520         my $confsrc = "$tmpdir/qemu-server.conf";
6521
6522         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
6523
6524         my $outfd = IO::File->new($tmpfn, "w") || die "unable to write config for VM $vmid\n";
6525
6526         my $cookie = { netcount => 0 };
6527         while (defined (my $line = <$srcfd>)) {
6528             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6529         }
6530
6531         $srcfd->close();
6532         $outfd->close();
6533     };
6534     if (my $err = $@) {
6535         unlink $tmpfn;
6536         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6537         die $err;
6538     }
6539
6540     rmtree $tmpdir;
6541
6542     rename $tmpfn, $conffile ||
6543         die "unable to commit configuration file '$conffile'\n";
6544
6545     PVE::Cluster::cfs_update(); # make sure we read new file
6546
6547     eval { rescan($vmid, 1); };
6548     warn $@ if $@;
6549 };
6550
6551 sub foreach_storage_used_by_vm {
6552     my ($conf, $func) = @_;
6553
6554     my $sidhash = {};
6555
6556     PVE::QemuConfig->foreach_volume($conf, sub {
6557         my ($ds, $drive) = @_;
6558         return if drive_is_cdrom($drive);
6559
6560         my $volid = $drive->{file};
6561
6562         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6563         $sidhash->{$sid} = $sid if $sid;
6564     });
6565
6566     foreach my $sid (sort keys %$sidhash) {
6567         &$func($sid);
6568     }
6569 }
6570
6571 my $qemu_snap_storage = {
6572     rbd => 1,
6573 };
6574 sub do_snapshots_with_qemu {
6575     my ($storecfg, $volid) = @_;
6576
6577     my $storage_name = PVE::Storage::parse_volume_id($volid);
6578     my $scfg = $storecfg->{ids}->{$storage_name};
6579     die "could not find storage '$storage_name'\n" if !defined($scfg);
6580
6581     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6582         return 1;
6583     }
6584
6585     if ($volid =~ m/\.(qcow2|qed)$/){
6586         return 1;
6587     }
6588
6589     return;
6590 }
6591
6592 sub qga_check_running {
6593     my ($vmid, $nowarn) = @_;
6594
6595     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6596     if ($@) {
6597         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6598         return 0;
6599     }
6600     return 1;
6601 }
6602
6603 sub template_create {
6604     my ($vmid, $conf, $disk) = @_;
6605
6606     my $storecfg = PVE::Storage::config();
6607
6608     PVE::QemuConfig->foreach_volume($conf, sub {
6609         my ($ds, $drive) = @_;
6610
6611         return if drive_is_cdrom($drive);
6612         return if $disk && $ds ne $disk;
6613
6614         my $volid = $drive->{file};
6615         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6616
6617         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6618         $drive->{file} = $voliddst;
6619         $conf->{$ds} = print_drive($drive);
6620         PVE::QemuConfig->write_config($vmid, $conf);
6621     });
6622 }
6623
6624 sub convert_iscsi_path {
6625     my ($path) = @_;
6626
6627     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6628         my $portal = $1;
6629         my $target = $2;
6630         my $lun = $3;
6631
6632         my $initiator_name = get_initiator_name();
6633
6634         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6635                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6636     }
6637
6638     die "cannot convert iscsi path '$path', unkown format\n";
6639 }
6640
6641 sub qemu_img_convert {
6642     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6643
6644     my $storecfg = PVE::Storage::config();
6645     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6646     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6647
6648     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6649
6650     my $cachemode;
6651     my $src_path;
6652     my $src_is_iscsi = 0;
6653     my $src_format;
6654
6655     if ($src_storeid) {
6656         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6657         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6658         $src_format = qemu_img_format($src_scfg, $src_volname);
6659         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6660         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6661         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6662     } elsif (-f $src_volid) {
6663         $src_path = $src_volid;
6664         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6665             $src_format = $1;
6666         }
6667     }
6668
6669     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6670
6671     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6672     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6673     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6674     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6675
6676     my $cmd = [];
6677     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6678     push @$cmd, '-l', "snapshot.name=$snapname"
6679         if $snapname && $src_format && $src_format eq "qcow2";
6680     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6681     push @$cmd, '-T', $cachemode if defined($cachemode);
6682
6683     if ($src_is_iscsi) {
6684         push @$cmd, '--image-opts';
6685         $src_path = convert_iscsi_path($src_path);
6686     } elsif ($src_format) {
6687         push @$cmd, '-f', $src_format;
6688     }
6689
6690     if ($dst_is_iscsi) {
6691         push @$cmd, '--target-image-opts';
6692         $dst_path = convert_iscsi_path($dst_path);
6693     } else {
6694         push @$cmd, '-O', $dst_format;
6695     }
6696
6697     push @$cmd, $src_path;
6698
6699     if (!$dst_is_iscsi && $is_zero_initialized) {
6700         push @$cmd, "zeroinit:$dst_path";
6701     } else {
6702         push @$cmd, $dst_path;
6703     }
6704
6705     my $parser = sub {
6706         my $line = shift;
6707         if($line =~ m/\((\S+)\/100\%\)/){
6708             my $percent = $1;
6709             my $transferred = int($size * $percent / 100);
6710             my $remaining = $size - $transferred;
6711
6712             print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6713         }
6714
6715     };
6716
6717     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6718     my $err = $@;
6719     die "copy failed: $err" if $err;
6720 }
6721
6722 sub qemu_img_format {
6723     my ($scfg, $volname) = @_;
6724
6725     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6726         return $1;
6727     } else {
6728         return "raw";
6729     }
6730 }
6731
6732 sub qemu_drive_mirror {
6733     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
6734
6735     $jobs = {} if !$jobs;
6736
6737     my $qemu_target;
6738     my $format;
6739     $jobs->{"drive-$drive"} = {};
6740
6741     if ($dst_volid =~ /^nbd:/) {
6742         $qemu_target = $dst_volid;
6743         $format = "nbd";
6744     } else {
6745         my $storecfg = PVE::Storage::config();
6746         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6747
6748         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6749
6750         $format = qemu_img_format($dst_scfg, $dst_volname);
6751
6752         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6753
6754         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6755     }
6756
6757     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6758     $opts->{format} = $format if $format;
6759
6760     if (defined($src_bitmap)) {
6761         $opts->{sync} = 'incremental';
6762         $opts->{bitmap} = $src_bitmap;
6763         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
6764     }
6765
6766     if (defined($bwlimit)) {
6767         $opts->{speed} = $bwlimit * 1024;
6768         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6769     } else {
6770         print "drive mirror is starting for drive-$drive\n";
6771     }
6772
6773     # if a job already runs for this device we get an error, catch it for cleanup
6774     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
6775     if (my $err = $@) {
6776         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6777         warn "$@\n" if $@;
6778         die "mirroring error: $err\n";
6779     }
6780
6781     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
6782 }
6783
6784 # $completion can be either
6785 # 'complete': wait until all jobs are ready, block-job-complete them (default)
6786 # 'cancel': wait until all jobs are ready, block-job-cancel them
6787 # 'skip': wait until all jobs are ready, return with block jobs in ready state
6788 sub qemu_drive_mirror_monitor {
6789     my ($vmid, $vmiddst, $jobs, $completion, $qga) = @_;
6790
6791     $completion //= 'complete';
6792
6793     eval {
6794         my $err_complete = 0;
6795
6796         while (1) {
6797             die "storage migration timed out\n" if $err_complete > 300;
6798
6799             my $stats = mon_cmd($vmid, "query-block-jobs");
6800
6801             my $running_mirror_jobs = {};
6802             foreach my $stat (@$stats) {
6803                 next if $stat->{type} ne 'mirror';
6804                 $running_mirror_jobs->{$stat->{device}} = $stat;
6805             }
6806
6807             my $readycounter = 0;
6808
6809             foreach my $job (keys %$jobs) {
6810
6811                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6812                     print "$job : finished\n";
6813                     delete $jobs->{$job};
6814                     next;
6815                 }
6816
6817                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6818
6819                 my $busy = $running_mirror_jobs->{$job}->{busy};
6820                 my $ready = $running_mirror_jobs->{$job}->{ready};
6821                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6822                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6823                     my $remaining = $total - $transferred;
6824                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6825
6826                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6827                 }
6828
6829                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6830             }
6831
6832             last if scalar(keys %$jobs) == 0;
6833
6834             if ($readycounter == scalar(keys %$jobs)) {
6835                 print "all mirroring jobs are ready \n";
6836                 last if $completion eq 'skip'; #do the complete later
6837
6838                 if ($vmiddst && $vmiddst != $vmid) {
6839                     my $agent_running = $qga && qga_check_running($vmid);
6840                     if ($agent_running) {
6841                         print "freeze filesystem\n";
6842                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6843                     } else {
6844                         print "suspend vm\n";
6845                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6846                     }
6847
6848                     # if we clone a disk for a new target vm, we don't switch the disk
6849                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6850
6851                     if ($agent_running) {
6852                         print "unfreeze filesystem\n";
6853                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6854                     } else {
6855                         print "resume vm\n";
6856                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6857                     }
6858
6859                     last;
6860                 } else {
6861
6862                     foreach my $job (keys %$jobs) {
6863                         # try to switch the disk if source and destination are on the same guest
6864                         print "$job: Completing block job...\n";
6865
6866                         my $op;
6867                         if ($completion eq 'complete') {
6868                             $op = 'block-job-complete';
6869                         } elsif ($completion eq 'cancel') {
6870                             $op = 'block-job-cancel';
6871                         } else {
6872                             die "invalid completion value: $completion\n";
6873                         }
6874                         eval { mon_cmd($vmid, $op, device => $job) };
6875                         if ($@ =~ m/cannot be completed/) {
6876                             print "$job: Block job cannot be completed, try again.\n";
6877                             $err_complete++;
6878                         }else {
6879                             print "$job: Completed successfully.\n";
6880                             $jobs->{$job}->{complete} = 1;
6881                         }
6882                     }
6883                 }
6884             }
6885             sleep 1;
6886         }
6887     };
6888     my $err = $@;
6889
6890     if ($err) {
6891         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6892         die "mirroring error: $err";
6893     }
6894
6895 }
6896
6897 sub qemu_blockjobs_cancel {
6898     my ($vmid, $jobs) = @_;
6899
6900     foreach my $job (keys %$jobs) {
6901         print "$job: Cancelling block job\n";
6902         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
6903         $jobs->{$job}->{cancel} = 1;
6904     }
6905
6906     while (1) {
6907         my $stats = mon_cmd($vmid, "query-block-jobs");
6908
6909         my $running_jobs = {};
6910         foreach my $stat (@$stats) {
6911             $running_jobs->{$stat->{device}} = $stat;
6912         }
6913
6914         foreach my $job (keys %$jobs) {
6915
6916             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6917                 print "$job: Done.\n";
6918                 delete $jobs->{$job};
6919             }
6920         }
6921
6922         last if scalar(keys %$jobs) == 0;
6923
6924         sleep 1;
6925     }
6926 }
6927
6928 sub clone_disk {
6929     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6930         $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
6931
6932     my $newvolid;
6933
6934     if (!$full) {
6935         print "create linked clone of drive $drivename ($drive->{file})\n";
6936         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6937         push @$newvollist, $newvolid;
6938     } else {
6939
6940         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6941         $storeid = $storage if $storage;
6942
6943         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6944
6945         print "create full clone of drive $drivename ($drive->{file})\n";
6946         my $name = undef;
6947         my $size = undef;
6948         if (drive_is_cloudinit($drive)) {
6949             $name = "vm-$newvmid-cloudinit";
6950             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6951             if ($scfg->{path}) {
6952                 $name .= ".$dst_format";
6953             }
6954             $snapname = undef;
6955             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
6956         } elsif ($drivename eq 'efidisk0') {
6957             $size = get_efivars_size($conf);
6958         } else {
6959             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
6960         }
6961         $newvolid = PVE::Storage::vdisk_alloc(
6962             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
6963         );
6964         push @$newvollist, $newvolid;
6965
6966         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6967
6968         if (drive_is_cloudinit($drive)) {
6969             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
6970             # if this is the case, we have to complete any block-jobs still there from
6971             # previous drive-mirrors
6972             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
6973                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
6974             }
6975             goto no_data_clone;
6976         }
6977
6978         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6979         if (!$running || $snapname) {
6980             # TODO: handle bwlimits
6981             if ($drivename eq 'efidisk0') {
6982                 # the relevant data on the efidisk may be smaller than the source
6983                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
6984                 # that is given by the OVMF_VARS.fd
6985                 my $src_path = PVE::Storage::path($storecfg, $drive->{file});
6986                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
6987                 run_command(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=1", "count=$size",
6988                     "if=$src_path", "of=$dst_path"]);
6989             } else {
6990                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6991             }
6992         } else {
6993
6994             my $kvmver = get_running_qemu_version ($vmid);
6995             if (!min_version($kvmver, 2, 7)) {
6996                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6997                     if $drive->{iothread};
6998             }
6999
7000             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7001                 $completion, $qga, $bwlimit);
7002         }
7003     }
7004
7005 no_data_clone:
7006     my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
7007
7008     my $disk = $drive;
7009     $disk->{format} = undef;
7010     $disk->{file} = $newvolid;
7011     $disk->{size} = $size if defined($size);
7012
7013     return $disk;
7014 }
7015
7016 sub get_running_qemu_version {
7017     my ($vmid) = @_;
7018     my $res = mon_cmd($vmid, "query-version");
7019     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7020 }
7021
7022 sub qemu_use_old_bios_files {
7023     my ($machine_type) = @_;
7024
7025     return if !$machine_type;
7026
7027     my $use_old_bios_files = undef;
7028
7029     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7030         $machine_type = $1;
7031         $use_old_bios_files = 1;
7032     } else {
7033         my $version = extract_version($machine_type, kvm_user_version());
7034         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7035         # load new efi bios files on migration. So this hack is required to allow
7036         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7037         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7038         $use_old_bios_files = !min_version($version, 2, 4);
7039     }
7040
7041     return ($use_old_bios_files, $machine_type);
7042 }
7043
7044 sub get_efivars_size {
7045     my ($conf) = @_;
7046     my $arch = get_vm_arch($conf);
7047     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7048     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
7049     return -s $ovmf_vars;
7050 }
7051
7052 sub update_efidisk_size {
7053     my ($conf) = @_;
7054
7055     return if !defined($conf->{efidisk0});
7056
7057     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
7058     $disk->{size} = get_efivars_size($conf);
7059     $conf->{efidisk0} = print_drive($disk);
7060
7061     return;
7062 }
7063
7064 sub create_efidisk($$$$$) {
7065     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7066
7067     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7068     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7069
7070     my $vars_size_b = -s $ovmf_vars;
7071     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7072     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7073     PVE::Storage::activate_volumes($storecfg, [$volid]);
7074
7075     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7076     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7077
7078     return ($volid, $size/1024);
7079 }
7080
7081 sub vm_iothreads_list {
7082     my ($vmid) = @_;
7083
7084     my $res = mon_cmd($vmid, 'query-iothreads');
7085
7086     my $iothreads = {};
7087     foreach my $iothread (@$res) {
7088         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7089     }
7090
7091     return $iothreads;
7092 }
7093
7094 sub scsihw_infos {
7095     my ($conf, $drive) = @_;
7096
7097     my $maxdev = 0;
7098
7099     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7100         $maxdev = 7;
7101     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7102         $maxdev = 1;
7103     } else {
7104         $maxdev = 256;
7105     }
7106
7107     my $controller = int($drive->{index} / $maxdev);
7108     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
7109         ? "virtioscsi"
7110         : "scsihw";
7111
7112     return ($maxdev, $controller, $controller_prefix);
7113 }
7114
7115 sub windows_version {
7116     my ($ostype) = @_;
7117
7118     return 0 if !$ostype;
7119
7120     my $winversion = 0;
7121
7122     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7123         $winversion = 5;
7124     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7125         $winversion = 6;
7126     } elsif ($ostype =~ m/^win(\d+)$/) {
7127         $winversion = $1;
7128     }
7129
7130     return $winversion;
7131 }
7132
7133 sub resolve_dst_disk_format {
7134         my ($storecfg, $storeid, $src_volname, $format) = @_;
7135         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7136
7137         if (!$format) {
7138             # if no target format is specified, use the source disk format as hint
7139             if ($src_volname) {
7140                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7141                 $format = qemu_img_format($scfg, $src_volname);
7142             } else {
7143                 return $defFormat;
7144             }
7145         }
7146
7147         # test if requested format is supported - else use default
7148         my $supported = grep { $_ eq $format } @$validFormats;
7149         $format = $defFormat if !$supported;
7150         return $format;
7151 }
7152
7153 # NOTE: if this logic changes, please update docs & possibly gui logic
7154 sub find_vmstate_storage {
7155     my ($conf, $storecfg) = @_;
7156
7157     # first, return storage from conf if set
7158     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7159
7160     my ($target, $shared, $local);
7161
7162     foreach_storage_used_by_vm($conf, sub {
7163         my ($sid) = @_;
7164         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7165         my $dst = $scfg->{shared} ? \$shared : \$local;
7166         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7167     });
7168
7169     # second, use shared storage where VM has at least one disk
7170     # third, use local storage where VM has at least one disk
7171     # fall back to local storage
7172     $target = $shared // $local // 'local';
7173
7174     return $target;
7175 }
7176
7177 sub generate_uuid {
7178     my ($uuid, $uuid_str);
7179     UUID::generate($uuid);
7180     UUID::unparse($uuid, $uuid_str);
7181     return $uuid_str;
7182 }
7183
7184 sub generate_smbios1_uuid {
7185     return "uuid=".generate_uuid();
7186 }
7187
7188 sub nbd_stop {
7189     my ($vmid) = @_;
7190
7191     mon_cmd($vmid, 'nbd-server-stop');
7192 }
7193
7194 sub create_reboot_request {
7195     my ($vmid) = @_;
7196     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7197         or die "failed to create reboot trigger file: $!\n";
7198     close($fh);
7199 }
7200
7201 sub clear_reboot_request {
7202     my ($vmid) = @_;
7203     my $path = "/run/qemu-server/$vmid.reboot";
7204     my $res = 0;
7205
7206     $res = unlink($path);
7207     die "could not remove reboot request for $vmid: $!"
7208         if !$res && $! != POSIX::ENOENT;
7209
7210     return $res;
7211 }
7212
7213 sub bootorder_from_legacy {
7214     my ($conf, $bootcfg) = @_;
7215
7216     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
7217     my $bootindex_hash = {};
7218     my $i = 1;
7219     foreach my $o (split(//, $boot)) {
7220         $bootindex_hash->{$o} = $i*100;
7221         $i++;
7222     }
7223
7224     my $bootorder = {};
7225
7226     PVE::QemuConfig->foreach_volume($conf, sub {
7227         my ($ds, $drive) = @_;
7228
7229         if (drive_is_cdrom ($drive, 1)) {
7230             if ($bootindex_hash->{d}) {
7231                 $bootorder->{$ds} = $bootindex_hash->{d};
7232                 $bootindex_hash->{d} += 1;
7233             }
7234         } elsif ($bootindex_hash->{c}) {
7235             $bootorder->{$ds} = $bootindex_hash->{c}
7236                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
7237             $bootindex_hash->{c} += 1;
7238         }
7239     });
7240
7241     if ($bootindex_hash->{n}) {
7242         for (my $i = 0; $i < $MAX_NETS; $i++) {
7243             my $netname = "net$i";
7244             next if !$conf->{$netname};
7245             $bootorder->{$netname} = $bootindex_hash->{n};
7246             $bootindex_hash->{n} += 1;
7247         }
7248     }
7249
7250     return $bootorder;
7251 }
7252
7253 # Generate default device list for 'boot: order=' property. Matches legacy
7254 # default boot order, but with explicit device names. This is important, since
7255 # the fallback for when neither 'order' nor the old format is specified relies
7256 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
7257 sub get_default_bootdevices {
7258     my ($conf) = @_;
7259
7260     my @ret = ();
7261
7262     # harddisk
7263     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
7264     push @ret, $first if $first;
7265
7266     # cdrom
7267     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
7268     push @ret, $first if $first;
7269
7270     # network
7271     for (my $i = 0; $i < $MAX_NETS; $i++) {
7272         my $netname = "net$i";
7273         next if !$conf->{$netname};
7274         push @ret, $netname;
7275         last;
7276     }
7277
7278     return \@ret;
7279 }
7280
7281 sub device_bootorder {
7282     my ($conf) = @_;
7283
7284     return bootorder_from_legacy($conf) if !defined($conf->{boot});
7285
7286     my $boot = parse_property_string($boot_fmt, $conf->{boot});
7287
7288     my $bootorder = {};
7289     if (!defined($boot) || $boot->{legacy}) {
7290         $bootorder = bootorder_from_legacy($conf, $boot);
7291     } elsif ($boot->{order}) {
7292         my $i = 100; # start at 100 to allow user to insert devices before us with -args
7293         for my $dev (PVE::Tools::split_list($boot->{order})) {
7294             $bootorder->{$dev} = $i++;
7295         }
7296     }
7297
7298     return $bootorder;
7299 }
7300
7301 # bash completion helper
7302
7303 sub complete_backup_archives {
7304     my ($cmdname, $pname, $cvalue) = @_;
7305
7306     my $cfg = PVE::Storage::config();
7307
7308     my $storeid;
7309
7310     if ($cvalue =~ m/^([^:]+):/) {
7311         $storeid = $1;
7312     }
7313
7314     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7315
7316     my $res = [];
7317     foreach my $id (keys %$data) {
7318         foreach my $item (@{$data->{$id}}) {
7319             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
7320             push @$res, $item->{volid} if defined($item->{volid});
7321         }
7322     }
7323
7324     return $res;
7325 }
7326
7327 my $complete_vmid_full = sub {
7328     my ($running) = @_;
7329
7330     my $idlist = vmstatus();
7331
7332     my $res = [];
7333
7334     foreach my $id (keys %$idlist) {
7335         my $d = $idlist->{$id};
7336         if (defined($running)) {
7337             next if $d->{template};
7338             next if $running && $d->{status} ne 'running';
7339             next if !$running && $d->{status} eq 'running';
7340         }
7341         push @$res, $id;
7342
7343     }
7344     return $res;
7345 };
7346
7347 sub complete_vmid {
7348     return &$complete_vmid_full();
7349 }
7350
7351 sub complete_vmid_stopped {
7352     return &$complete_vmid_full(0);
7353 }
7354
7355 sub complete_vmid_running {
7356     return &$complete_vmid_full(1);
7357 }
7358
7359 sub complete_storage {
7360
7361     my $cfg = PVE::Storage::config();
7362     my $ids = $cfg->{ids};
7363
7364     my $res = [];
7365     foreach my $sid (keys %$ids) {
7366         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7367         next if !$ids->{$sid}->{content}->{images};
7368         push @$res, $sid;
7369     }
7370
7371     return $res;
7372 }
7373
7374 sub complete_migration_storage {
7375     my ($cmd, $param, $current_value, $all_args) = @_;
7376
7377     my $targetnode = @$all_args[1];
7378
7379     my $cfg = PVE::Storage::config();
7380     my $ids = $cfg->{ids};
7381
7382     my $res = [];
7383     foreach my $sid (keys %$ids) {
7384         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7385         next if !$ids->{$sid}->{content}->{images};
7386         push @$res, $sid;
7387     }
7388
7389     return $res;
7390 }
7391
7392 1;