PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::CGroup;
  31 use PVE::DataCenterConfig;
  32 use PVE::Exception qw(raise raise_param_exc);
  33 use PVE::Format qw(render_duration render_bytes);
  34 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  35 use PVE::INotify;
  36 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  37 use PVE::ProcFSTools;
  38 use PVE::PBSClient;
  39 use PVE::RPCEnvironment;
  40 use PVE::Storage;
  41 use PVE::SysFSTools;
  42 use PVE::Systemd;
  43 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  44
  45 use PVE::QMPClient;
  46 use PVE::QemuConfig;
  47 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  48 use PVE::QemuServer::Cloudinit;
  49 use PVE::QemuServer::CGroup;
  50 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  51 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  52 use PVE::QemuServer::Machine;
  53 use PVE::QemuServer::Memory;
  54 use PVE::QemuServer::Monitor qw(mon_cmd);
  55 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  56 use PVE::QemuServer::USB qw(parse_usb_device);
  57
  58 my $have_sdn;
  59 eval {
  60     require PVE::Network::SDN::Zones;
  61     $have_sdn = 1;
  62 };
  63
  64 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  65 my $OVMF = {
  66     x86_64 => [
  67         "$EDK2_FW_BASE/OVMF_CODE.fd",
  68         "$EDK2_FW_BASE/OVMF_VARS.fd"
  69     ],
  70     aarch64 => [
  71         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  72         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  73     ],
  74 };
  75
  76 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  77
  78 # Note about locking: we use flock on the config file protect
  79 # against concurent actions.
  80 # Aditionaly, we have a 'lock' setting in the config file. This
  81 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  82 # allowed when such lock is set. But you can ignore this kind of
  83 # lock with the --skiplock flag.
  84
  85 cfs_register_file('/qemu-server/',
  86                   \&parse_vm_config,
  87                   \&write_vm_config);
  88
  89 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  90     description => "Some command save/restore state from this location.",
  91     type => 'string',
  92     maxLength => 128,
  93     optional => 1,
  94 });
  95
  96 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  97         description => "Specifies the Qemu machine type.",
  98         type => 'string',
  99         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 100         maxLength => 40,
 101         optional => 1,
 102 });
 103
 104
 105 sub map_storage {
 106     my ($map, $source) = @_;
 107
 108     return $source if !defined($map);
 109
 110     return $map->{entries}->{$source}
 111         if $map->{entries} && defined($map->{entries}->{$source});
 112
 113     return $map->{default} if $map->{default};
 114
 115     # identity (fallback)
 116     return $source;
 117 }
 118
 119 PVE::JSONSchema::register_standard_option('pve-targetstorage', {
 120     description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
 121     type => 'string',
 122     format => 'storagepair-list',
 123     optional => 1,
 124 });
 125
 126 #no warnings 'redefine';
 127
 128 my $nodename_cache;
 129 sub nodename {
 130     $nodename_cache //= PVE::INotify::nodename();
 131     return $nodename_cache;
 132 }
 133
 134 my $watchdog_fmt = {
 135     model => {
 136         default_key => 1,
 137         type => 'string',
 138         enum => [qw(i6300esb ib700)],
 139         description => "Watchdog type to emulate.",
 140         default => 'i6300esb',
 141         optional => 1,
 142     },
 143     action => {
 144         type => 'string',
 145         enum => [qw(reset shutdown poweroff pause debug none)],
 146         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 147         optional => 1,
 148     },
 149 };
 150 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 151
 152 my $agent_fmt = {
 153     enabled => {
 154         description => "Enable/disable Qemu GuestAgent.",
 155         type => 'boolean',
 156         default => 0,
 157         default_key => 1,
 158     },
 159     fstrim_cloned_disks => {
 160         description => "Run fstrim after moving a disk or migrating the VM.",
 161         type => 'boolean',
 162         optional => 1,
 163         default => 0
 164     },
 165     type => {
 166         description => "Select the agent type",
 167         type => 'string',
 168         default => 'virtio',
 169         optional => 1,
 170         enum => [qw(virtio isa)],
 171     },
 172 };
 173
 174 my $vga_fmt = {
 175     type => {
 176         description => "Select the VGA type.",
 177         type => 'string',
 178         default => 'std',
 179         optional => 1,
 180         default_key => 1,
 181         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 182     },
 183     memory => {
 184         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 185         type => 'integer',
 186         optional => 1,
 187         minimum => 4,
 188         maximum => 512,
 189     },
 190 };
 191
 192 my $ivshmem_fmt = {
 193     size => {
 194         type => 'integer',
 195         minimum => 1,
 196         description => "The size of the file in MB.",
 197     },
 198     name => {
 199         type => 'string',
 200         pattern => '[a-zA-Z0-9\-]+',
 201         optional => 1,
 202         format_description => 'string',
 203         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 204     },
 205 };
 206
 207 my $audio_fmt = {
 208     device => {
 209         type => 'string',
 210         enum => [qw(ich9-intel-hda intel-hda AC97)],
 211         description =>  "Configure an audio device."
 212     },
 213     driver =>  {
 214         type => 'string',
 215         enum => ['spice', 'none'],
 216         default => 'spice',
 217         optional => 1,
 218         description => "Driver backend for the audio device."
 219     },
 220 };
 221
 222 my $spice_enhancements_fmt = {
 223     foldersharing => {
 224         type => 'boolean',
 225         optional => 1,
 226         default => '0',
 227         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 228     },
 229     videostreaming =>  {
 230         type => 'string',
 231         enum => ['off', 'all', 'filter'],
 232         default => 'off',
 233         optional => 1,
 234         description => "Enable video streaming. Uses compression for detected video streams."
 235     },
 236 };
 237
 238 my $rng_fmt = {
 239     source => {
 240         type => 'string',
 241         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 242         default_key => 1,
 243         description => "The file on the host to gather entropy from. In most"
 244                      . " cases /dev/urandom should be preferred over /dev/random"
 245                      . " to avoid entropy-starvation issues on the host. Using"
 246                      . " urandom does *not* decrease security in any meaningful"
 247                      . " way, as it's still seeded from real entropy, and the"
 248                      . " bytes provided will most likely be mixed with real"
 249                      . " entropy on the guest as well. /dev/hwrng can be used"
 250                      . " to pass through a hardware RNG from the host.",
 251     },
 252     max_bytes => {
 253         type => 'integer',
 254         description => "Maximum bytes of entropy injected into the guest every"
 255                      . " 'period' milliseconds. Prefer a lower value when using"
 256                      . " /dev/random as source. Use 0 to disable limiting"
 257                      . " (potentially dangerous!).",
 258         optional => 1,
 259
 260         # default is 1 KiB/s, provides enough entropy to the guest to avoid
 261         # boot-starvation issues (e.g. systemd etc...) while allowing no chance
 262         # of overwhelming the host, provided we're reading from /dev/urandom
 263         default => 1024,
 264     },
 265     period => {
 266         type => 'integer',
 267         description => "Every 'period' milliseconds the entropy-injection quota"
 268                      . " is reset, allowing the guest to retrieve another"
 269                      . " 'max_bytes' of entropy.",
 270         optional => 1,
 271         default => 1000,
 272     },
 273 };
 274
 275 my $confdesc = {
 276     onboot => {
 277         optional => 1,
 278         type => 'boolean',
 279         description => "Specifies whether a VM will be started during system bootup.",
 280         default => 0,
 281     },
 282     autostart => {
 283         optional => 1,
 284         type => 'boolean',
 285         description => "Automatic restart after crash (currently ignored).",
 286         default => 0,
 287     },
 288     hotplug => {
 289         optional => 1,
 290         type => 'string', format => 'pve-hotplug-features',
 291         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 292         default => 'network,disk,usb',
 293     },
 294     reboot => {
 295         optional => 1,
 296         type => 'boolean',
 297         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 298         default => 1,
 299     },
 300     lock => {
 301         optional => 1,
 302         type => 'string',
 303         description => "Lock/unlock the VM.",
 304         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 305     },
 306     cpulimit => {
 307         optional => 1,
 308         type => 'number',
 309         description => "Limit of CPU usage.",
 310         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 311         minimum => 0,
 312         maximum => 128,
 313         default => 0,
 314     },
 315     cpuunits => {
 316         optional => 1,
 317         type => 'integer',
 318         description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
 319         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
 320             ." The larger the number is, the more CPU time this VM gets. Number is relative to"
 321             ." weights of all the other running VMs.",
 322         minimum => 2,
 323         maximum => 262144,
 324         default => 'cgroup v1: 1024, cgroup v2: 100',
 325     },
 326     memory => {
 327         optional => 1,
 328         type => 'integer',
 329         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 330         minimum => 16,
 331         default => 512,
 332     },
 333     balloon => {
 334         optional => 1,
 335         type => 'integer',
 336         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 337         minimum => 0,
 338     },
 339     shares => {
 340         optional => 1,
 341         type => 'integer',
 342         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 343         minimum => 0,
 344         maximum => 50000,
 345         default => 1000,
 346     },
 347     keyboard => {
 348         optional => 1,
 349         type => 'string',
 350         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 351                        "It should not be necessary to set it.",
 352         enum => PVE::Tools::kvmkeymaplist(),
 353         default => undef,
 354     },
 355     name => {
 356         optional => 1,
 357         type => 'string', format => 'dns-name',
 358         description => "Set a name for the VM. Only used on the configuration web interface.",
 359     },
 360     scsihw => {
 361         optional => 1,
 362         type => 'string',
 363         description => "SCSI controller model",
 364         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 365         default => 'lsi',
 366     },
 367     description => {
 368         optional => 1,
 369         type => 'string',
 370         description => "Description for the VM. Shown in the web-interface VM's summary."
 371             ." This is saved as comment inside the configuration file.",
 372         maxLength => 1024 * 8,
 373     },
 374     ostype => {
 375         optional => 1,
 376         type => 'string',
 377         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 378         description => "Specify guest operating system.",
 379         verbose_description => <<EODESC,
 380 Specify guest operating system. This is used to enable special
 381 optimization/features for specific operating systems:
 382
 383 [horizontal]
 384 other;; unspecified OS
 385 wxp;; Microsoft Windows XP
 386 w2k;; Microsoft Windows 2000
 387 w2k3;; Microsoft Windows 2003
 388 w2k8;; Microsoft Windows 2008
 389 wvista;; Microsoft Windows Vista
 390 win7;; Microsoft Windows 7
 391 win8;; Microsoft Windows 8/2012/2012r2
 392 win10;; Microsoft Windows 10/2016/2019
 393 l24;; Linux 2.4 Kernel
 394 l26;; Linux 2.6 - 5.X Kernel
 395 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 396 EODESC
 397     },
 398     boot => {
 399         optional => 1,
 400         type => 'string', format => 'pve-qm-boot',
 401         description => "Specify guest boot order. Use with 'order=', usage with"
 402                      . " no key or 'legacy=' is deprecated.",
 403     },
 404     bootdisk => {
 405         optional => 1,
 406         type => 'string', format => 'pve-qm-bootdisk',
 407         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 408         pattern => '(ide|sata|scsi|virtio)\d+',
 409     },
 410     smp => {
 411         optional => 1,
 412         type => 'integer',
 413         description => "The number of CPUs. Please use option -sockets instead.",
 414         minimum => 1,
 415         default => 1,
 416     },
 417     sockets => {
 418         optional => 1,
 419         type => 'integer',
 420         description => "The number of CPU sockets.",
 421         minimum => 1,
 422         default => 1,
 423     },
 424     cores => {
 425         optional => 1,
 426         type => 'integer',
 427         description => "The number of cores per socket.",
 428         minimum => 1,
 429         default => 1,
 430     },
 431     numa => {
 432         optional => 1,
 433         type => 'boolean',
 434         description => "Enable/disable NUMA.",
 435         default => 0,
 436     },
 437     hugepages => {
 438         optional => 1,
 439         type => 'string',
 440         description => "Enable/disable hugepages memory.",
 441         enum => [qw(any 2 1024)],
 442     },
 443     keephugepages => {
 444         optional => 1,
 445         type => 'boolean',
 446         default => 0,
 447         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 448             ." after VM shutdown and can be used for subsequent starts.",
 449     },
 450     vcpus => {
 451         optional => 1,
 452         type => 'integer',
 453         description => "Number of hotplugged vcpus.",
 454         minimum => 1,
 455         default => 0,
 456     },
 457     acpi => {
 458         optional => 1,
 459         type => 'boolean',
 460         description => "Enable/disable ACPI.",
 461         default => 1,
 462     },
 463     agent => {
 464         optional => 1,
 465         description => "Enable/disable Qemu GuestAgent and its properties.",
 466         type => 'string',
 467         format => $agent_fmt,
 468     },
 469     kvm => {
 470         optional => 1,
 471         type => 'boolean',
 472         description => "Enable/disable KVM hardware virtualization.",
 473         default => 1,
 474     },
 475     tdf => {
 476         optional => 1,
 477         type => 'boolean',
 478         description => "Enable/disable time drift fix.",
 479         default => 0,
 480     },
 481     localtime => {
 482         optional => 1,
 483         type => 'boolean',
 484         description => "Set the real time clock to local time. This is enabled by default if ostype"
 485             ." indicates a Microsoft OS.",
 486     },
 487     freeze => {
 488         optional => 1,
 489         type => 'boolean',
 490         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 491     },
 492     vga => {
 493         optional => 1,
 494         type => 'string', format => $vga_fmt,
 495         description => "Configure the VGA hardware.",
 496         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 497             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 498             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 499             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 500             ." display server. For win* OS you can select how many independent displays you want,"
 501             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 502             ." using a serial device as terminal.",
 503     },
 504     watchdog => {
 505         optional => 1,
 506         type => 'string', format => 'pve-qm-watchdog',
 507         description => "Create a virtual hardware watchdog device.",
 508         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 509             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 510             ." else the watchdog will reset the guest (or execute the respective action specified)",
 511     },
 512     startdate => {
 513         optional => 1,
 514         type => 'string',
 515         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 516         description => "Set the initial date of the real time clock. Valid format for date are:"
 517             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 518         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 519         default => 'now',
 520     },
 521     startup =>  get_standard_option('pve-startup-order'),
 522     template => {
 523         optional => 1,
 524         type => 'boolean',
 525         description => "Enable/disable Template.",
 526         default => 0,
 527     },
 528     args => {
 529         optional => 1,
 530         type => 'string',
 531         description => "Arbitrary arguments passed to kvm.",
 532         verbose_description => <<EODESCR,
 533 Arbitrary arguments passed to kvm, for example:
 534
 535 args: -no-reboot -no-hpet
 536
 537 NOTE: this option is for experts only.
 538 EODESCR
 539     },
 540     tablet => {
 541         optional => 1,
 542         type => 'boolean',
 543         default => 1,
 544         description => "Enable/disable the USB tablet device.",
 545         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 546             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 547             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 548             ." may consider disabling this to save some context switches. This is turned off by"
 549             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 550     },
 551     migrate_speed => {
 552         optional => 1,
 553         type => 'integer',
 554         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 555         minimum => 0,
 556         default => 0,
 557     },
 558     migrate_downtime => {
 559         optional => 1,
 560         type => 'number',
 561         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 562         minimum => 0,
 563         default => 0.1,
 564     },
 565     cdrom => {
 566         optional => 1,
 567         type => 'string', format => 'pve-qm-ide',
 568         typetext => '<volume>',
 569         description => "This is an alias for option -ide2",
 570     },
 571     cpu => {
 572         optional => 1,
 573         description => "Emulated CPU type.",
 574         type => 'string',
 575         format => 'pve-vm-cpu-conf',
 576     },
 577     parent => get_standard_option('pve-snapshot-name', {
 578         optional => 1,
 579         description => "Parent snapshot name. This is used internally, and should not be modified.",
 580     }),
 581     snaptime => {
 582         optional => 1,
 583         description => "Timestamp for snapshots.",
 584         type => 'integer',
 585         minimum => 0,
 586     },
 587     vmstate => {
 588         optional => 1,
 589         type => 'string', format => 'pve-volume-id',
 590         description => "Reference to a volume which stores the VM state. This is used internally"
 591             ." for snapshots.",
 592     },
 593     vmstatestorage => get_standard_option('pve-storage-id', {
 594         description => "Default storage for VM state volumes/files.",
 595         optional => 1,
 596     }),
 597     runningmachine => get_standard_option('pve-qemu-machine', {
 598         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 599             ." for snapshots.",
 600     }),
 601     runningcpu => {
 602         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 603             ." internally for snapshots.",
 604         optional => 1,
 605         type => 'string',
 606         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 607         format_description => 'QEMU -cpu parameter'
 608     },
 609     machine => get_standard_option('pve-qemu-machine'),
 610     arch => {
 611         description => "Virtual processor architecture. Defaults to the host.",
 612         optional => 1,
 613         type => 'string',
 614         enum => [qw(x86_64 aarch64)],
 615     },
 616     smbios1 => {
 617         description => "Specify SMBIOS type 1 fields.",
 618         type => 'string', format => 'pve-qm-smbios1',
 619         maxLength => 512,
 620         optional => 1,
 621     },
 622     protection => {
 623         optional => 1,
 624         type => 'boolean',
 625         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 626             ." remove disk operations.",
 627         default => 0,
 628     },
 629     bios => {
 630         optional => 1,
 631         type => 'string',
 632         enum => [ qw(seabios ovmf) ],
 633         description => "Select BIOS implementation.",
 634         default => 'seabios',
 635     },
 636     vmgenid => {
 637         type => 'string',
 638         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 639         format_description => 'UUID',
 640         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 641             ." to disable explicitly.",
 642         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 643             ." value identifier to the guest OS. This allows to notify the guest operating system"
 644             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 645             ." execution or creation from a template). The guest operating system notices the"
 646             ." change, and is then able to react as appropriate by marking its copies of"
 647             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 648             ."Note that auto-creation only works when done through API/CLI create or update methods"
 649             .", but not when manually editing the config file.",
 650         default => "1 (autogenerated)",
 651         optional => 1,
 652     },
 653     hookscript => {
 654         type => 'string',
 655         format => 'pve-volume-id',
 656         optional => 1,
 657         description => "Script that will be executed during various steps in the vms lifetime.",
 658     },
 659     ivshmem => {
 660         type => 'string',
 661         format => $ivshmem_fmt,
 662         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 663             ." the host.",
 664         optional => 1,
 665     },
 666     audio0 => {
 667         type => 'string',
 668         format => $audio_fmt,
 669         description => "Configure a audio device, useful in combination with QXL/Spice.",
 670         optional => 1
 671     },
 672     spice_enhancements => {
 673         type => 'string',
 674         format => $spice_enhancements_fmt,
 675         description => "Configure additional enhancements for SPICE.",
 676         optional => 1
 677     },
 678     tags => {
 679         type => 'string', format => 'pve-tag-list',
 680         description => 'Tags of the VM. This is only meta information.',
 681         optional => 1,
 682     },
 683     rng0 => {
 684         type => 'string',
 685         format => $rng_fmt,
 686         description => "Configure a VirtIO-based Random Number Generator.",
 687         optional => 1,
 688     },
 689 };
 690
 691 my $cicustom_fmt = {
 692     meta => {
 693         type => 'string',
 694         optional => 1,
 695         description => 'Specify a custom file containing all meta data passed to the VM via"
 696             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 697         format => 'pve-volume-id',
 698         format_description => 'volume',
 699     },
 700     network => {
 701         type => 'string',
 702         optional => 1,
 703         description => 'Specify a custom file containing all network data passed to the VM via'
 704             .' cloud-init.',
 705         format => 'pve-volume-id',
 706         format_description => 'volume',
 707     },
 708     user => {
 709         type => 'string',
 710         optional => 1,
 711         description => 'Specify a custom file containing all user data passed to the VM via'
 712             .' cloud-init.',
 713         format => 'pve-volume-id',
 714         format_description => 'volume',
 715     },
 716 };
 717 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 718
 719 my $confdesc_cloudinit = {
 720     citype => {
 721         optional => 1,
 722         type => 'string',
 723         description => 'Specifies the cloud-init configuration format. The default depends on the'
 724             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 725             .' and `configdrive2` for windows.',
 726         enum => ['configdrive2', 'nocloud', 'opennebula'],
 727     },
 728     ciuser => {
 729         optional => 1,
 730         type => 'string',
 731         description => "cloud-init: User name to change ssh keys and password for instead of the"
 732             ." image's configured default user.",
 733     },
 734     cipassword => {
 735         optional => 1,
 736         type => 'string',
 737         description => 'cloud-init: Password to assign the user. Using this is generally not'
 738             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 739             .' support hashed passwords.',
 740     },
 741     cicustom => {
 742         optional => 1,
 743         type => 'string',
 744         description => 'cloud-init: Specify custom files to replace the automatically generated'
 745             .' ones at start.',
 746         format => 'pve-qm-cicustom',
 747     },
 748     searchdomain => {
 749         optional => 1,
 750         type => 'string',
 751         description => "cloud-init: Sets DNS search domains for a container. Create will'
 752             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 753             .' are set.",
 754     },
 755     nameserver => {
 756         optional => 1,
 757         type => 'string', format => 'address-list',
 758         description => "cloud-init: Sets DNS server IP address for a container. Create will'
 759             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 760             .' are set.",
 761     },
 762     sshkeys => {
 763         optional => 1,
 764         type => 'string',
 765         format => 'urlencoded',
 766         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 767     },
 768 };
 769
 770 # what about other qemu settings ?
 771 #cpu => 'string',
 772 #machine => 'string',
 773 #fda => 'file',
 774 #fdb => 'file',
 775 #mtdblock => 'file',
 776 #sd => 'file',
 777 #pflash => 'file',
 778 #snapshot => 'bool',
 779 #bootp => 'file',
 780 ##tftp => 'dir',
 781 ##smb => 'dir',
 782 #kernel => 'file',
 783 #append => 'string',
 784 #initrd => 'file',
 785 ##soundhw => 'string',
 786
 787 while (my ($k, $v) = each %$confdesc) {
 788     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 789 }
 790
 791 my $MAX_USB_DEVICES = 5;
 792 my $MAX_NETS = 32;
 793 my $MAX_SERIAL_PORTS = 4;
 794 my $MAX_PARALLEL_PORTS = 3;
 795 my $MAX_NUMA = 8;
 796
 797 my $numa_fmt = {
 798     cpus => {
 799         type => "string",
 800         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 801         description => "CPUs accessing this NUMA node.",
 802         format_description => "id[-id];...",
 803     },
 804     memory => {
 805         type => "number",
 806         description => "Amount of memory this NUMA node provides.",
 807         optional => 1,
 808     },
 809     hostnodes => {
 810         type => "string",
 811         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 812         description => "Host NUMA nodes to use.",
 813         format_description => "id[-id];...",
 814         optional => 1,
 815     },
 816     policy => {
 817         type => 'string',
 818         enum => [qw(preferred bind interleave)],
 819         description => "NUMA allocation policy.",
 820         optional => 1,
 821     },
 822 };
 823 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 824 my $numadesc = {
 825     optional => 1,
 826     type => 'string', format => $numa_fmt,
 827     description => "NUMA topology.",
 828 };
 829 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 830
 831 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 832     $confdesc->{"numa$i"} = $numadesc;
 833 }
 834
 835 my $nic_model_list = [
 836     'e1000',
 837     'e1000-82540em',
 838     'e1000-82544gc',
 839     'e1000-82545em',
 840     'e1000e',
 841     'i82551',
 842     'i82557b',
 843     'i82559er',
 844     'ne2k_isa',
 845     'ne2k_pci',
 846     'pcnet',
 847     'rtl8139',
 848     'virtio',
 849     'vmxnet3',
 850 ];
 851 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 852
 853 my $net_fmt_bridge_descr = <<__EOD__;
 854 Bridge to attach the network device to. The Proxmox VE standard bridge
 855 is called 'vmbr0'.
 856
 857 If you do not specify a bridge, we create a kvm user (NATed) network
 858 device, which provides DHCP and DNS services. The following addresses
 859 are used:
 860
 861  10.0.2.2   Gateway
 862  10.0.2.3   DNS Server
 863  10.0.2.4   SMB Server
 864
 865 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 866 __EOD__
 867
 868 my $net_fmt = {
 869     macaddr  => get_standard_option('mac-addr', {
 870         description => "MAC address. That address must be unique withing your network. This is"
 871             ." automatically generated if not specified.",
 872     }),
 873     model => {
 874         type => 'string',
 875         description => "Network Card Model. The 'virtio' model provides the best performance with"
 876             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 877             ." best to use 'e1000'.",
 878         enum => $nic_model_list,
 879         default_key => 1,
 880     },
 881     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 882     bridge => {
 883         type => 'string',
 884         description => $net_fmt_bridge_descr,
 885         format_description => 'bridge',
 886         pattern => '[-_.\w\d]+',
 887         optional => 1,
 888     },
 889     queues => {
 890         type => 'integer',
 891         minimum => 0, maximum => 16,
 892         description => 'Number of packet queues to be used on the device.',
 893         optional => 1,
 894     },
 895     rate => {
 896         type => 'number',
 897         minimum => 0,
 898         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 899         optional => 1,
 900     },
 901     tag => {
 902         type => 'integer',
 903         minimum => 1, maximum => 4094,
 904         description => 'VLAN tag to apply to packets on this interface.',
 905         optional => 1,
 906     },
 907     trunks => {
 908         type => 'string',
 909         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 910         description => 'VLAN trunks to pass through this interface.',
 911         format_description => 'vlanid[;vlanid...]',
 912         optional => 1,
 913     },
 914     firewall => {
 915         type => 'boolean',
 916         description => 'Whether this interface should be protected by the firewall.',
 917         optional => 1,
 918     },
 919     link_down => {
 920         type => 'boolean',
 921         description => 'Whether this interface should be disconnected (like pulling the plug).',
 922         optional => 1,
 923     },
 924     mtu => {
 925         type => 'integer',
 926         minimum => 1, maximum => 65520,
 927         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 928         optional => 1,
 929     },
 930 };
 931
 932 my $netdesc = {
 933     optional => 1,
 934     type => 'string', format => $net_fmt,
 935     description => "Specify network devices.",
 936 };
 937
 938 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 939
 940 my $ipconfig_fmt = {
 941     ip => {
 942         type => 'string',
 943         format => 'pve-ipv4-config',
 944         format_description => 'IPv4Format/CIDR',
 945         description => 'IPv4 address in CIDR format.',
 946         optional => 1,
 947         default => 'dhcp',
 948     },
 949     gw => {
 950         type => 'string',
 951         format => 'ipv4',
 952         format_description => 'GatewayIPv4',
 953         description => 'Default gateway for IPv4 traffic.',
 954         optional => 1,
 955         requires => 'ip',
 956     },
 957     ip6 => {
 958         type => 'string',
 959         format => 'pve-ipv6-config',
 960         format_description => 'IPv6Format/CIDR',
 961         description => 'IPv6 address in CIDR format.',
 962         optional => 1,
 963         default => 'dhcp',
 964     },
 965     gw6 => {
 966         type => 'string',
 967         format => 'ipv6',
 968         format_description => 'GatewayIPv6',
 969         description => 'Default gateway for IPv6 traffic.',
 970         optional => 1,
 971         requires => 'ip6',
 972     },
 973 };
 974 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 975 my $ipconfigdesc = {
 976     optional => 1,
 977     type => 'string', format => 'pve-qm-ipconfig',
 978     description => <<'EODESCR',
 979 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 980
 981 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 982
 983 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
 984 gateway should be provided.
 985 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
 986 cloud-init 19.4 or newer.
 987
 988 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
 989 dhcp on IPv4.
 990 EODESCR
 991 };
 992 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 993
 994 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 995     $confdesc->{"net$i"} = $netdesc;
 996     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 997 }
 998
 999 foreach my $key (keys %$confdesc_cloudinit) {
1000     $confdesc->{$key} = $confdesc_cloudinit->{$key};
1001 }
1002
1003 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1004 sub verify_volume_id_or_qm_path {
1005     my ($volid, $noerr) = @_;
1006
1007     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
1008         return $volid;
1009     }
1010
1011     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
1012     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1013     if ($@) {
1014         return if $noerr;
1015         die $@;
1016     }
1017     return $volid;
1018 }
1019
1020 my $usb_fmt = {
1021     host => {
1022         default_key => 1,
1023         type => 'string', format => 'pve-qm-usb-device',
1024         format_description => 'HOSTUSBDEVICE|spice',
1025         description => <<EODESCR,
1026 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1027
1028  'bus-port(.port)*' (decimal numbers) or
1029  'vendor_id:product_id' (hexadeciaml numbers) or
1030  'spice'
1031
1032 You can use the 'lsusb -t' command to list existing usb devices.
1033
1034 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1035 machines - use with special care.
1036
1037 The value 'spice' can be used to add a usb redirection devices for spice.
1038 EODESCR
1039     },
1040     usb3 => {
1041         optional => 1,
1042         type => 'boolean',
1043         description => "Specifies whether if given host option is a USB3 device or port.",
1044         default => 0,
1045     },
1046 };
1047
1048 my $usbdesc = {
1049     optional => 1,
1050     type => 'string', format => $usb_fmt,
1051     description => "Configure an USB device (n is 0 to 4).",
1052 };
1053 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1054
1055 my $serialdesc = {
1056         optional => 1,
1057         type => 'string',
1058         pattern => '(/dev/.+|socket)',
1059         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1060         verbose_description =>  <<EODESCR,
1061 Create a serial device inside the VM (n is 0 to 3), and pass through a
1062 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1063 host side (use 'qm terminal' to open a terminal connection).
1064
1065 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1066 use with special care.
1067
1068 CAUTION: Experimental! User reported problems with this option.
1069 EODESCR
1070 };
1071
1072 my $paralleldesc= {
1073         optional => 1,
1074         type => 'string',
1075         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1076         description =>  "Map host parallel devices (n is 0 to 2).",
1077         verbose_description =>  <<EODESCR,
1078 Map host parallel devices (n is 0 to 2).
1079
1080 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1081 machines - use with special care.
1082
1083 CAUTION: Experimental! User reported problems with this option.
1084 EODESCR
1085 };
1086
1087 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1088     $confdesc->{"parallel$i"} = $paralleldesc;
1089 }
1090
1091 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1092     $confdesc->{"serial$i"} = $serialdesc;
1093 }
1094
1095 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1096     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1097 }
1098
1099 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1100     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1101 }
1102
1103 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1104     $confdesc->{"usb$i"} = $usbdesc;
1105 }
1106
1107 my $boot_fmt = {
1108     legacy => {
1109         optional => 1,
1110         default_key => 1,
1111         type => 'string',
1112         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1113                      . " Deprecated, use 'order=' instead.",
1114         pattern => '[acdn]{1,4}',
1115         format_description => "[acdn]{1,4}",
1116
1117         # note: this is also the fallback if boot: is not given at all
1118         default => 'cdn',
1119     },
1120     order => {
1121         optional => 1,
1122         type => 'string',
1123         format => 'pve-qm-bootdev-list',
1124         format_description => "device[;device...]",
1125         description => <<EODESC,
1126 The guest will attempt to boot from devices in the order they appear here.
1127
1128 Disks, optical drives and passed-through storage USB devices will be directly
1129 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1130 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1131
1132 Note that only devices in this list will be marked as bootable and thus loaded
1133 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1134 (e.g. software-raid), you need to specify all of them here.
1135
1136 Overrides the deprecated 'legacy=[acdn]*' value when given.
1137 EODESC
1138     },
1139 };
1140 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1141
1142 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1143 sub verify_bootdev {
1144     my ($dev, $noerr) = @_;
1145
1146     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && $dev !~ m/^efidisk/;
1147
1148     my $check = sub {
1149         my ($base) = @_;
1150         return 0 if $dev !~ m/^$base\d+$/;
1151         return 0 if !$confdesc->{$dev};
1152         return 1;
1153     };
1154
1155     return $dev if $check->("net");
1156     return $dev if $check->("usb");
1157     return $dev if $check->("hostpci");
1158
1159     return if $noerr;
1160     die "invalid boot device '$dev'\n";
1161 }
1162
1163 sub print_bootorder {
1164     my ($devs) = @_;
1165     return "" if !@$devs;
1166     my $data = { order => join(';', @$devs) };
1167     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1168 }
1169
1170 my $kvm_api_version = 0;
1171
1172 sub kvm_version {
1173     return $kvm_api_version if $kvm_api_version;
1174
1175     open my $fh, '<', '/dev/kvm' or return;
1176
1177     # 0xae00 => KVM_GET_API_VERSION
1178     $kvm_api_version = ioctl($fh, 0xae00, 0);
1179     close($fh);
1180
1181     return $kvm_api_version;
1182 }
1183
1184 my $kvm_user_version = {};
1185 my $kvm_mtime = {};
1186
1187 sub kvm_user_version {
1188     my ($binary) = @_;
1189
1190     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1191     my $st = stat($binary);
1192
1193     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1194     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1195         $cachedmtime == $st->mtime;
1196
1197     $kvm_user_version->{$binary} = 'unknown';
1198     $kvm_mtime->{$binary} = $st->mtime;
1199
1200     my $code = sub {
1201         my $line = shift;
1202         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1203             $kvm_user_version->{$binary} = $2;
1204         }
1205     };
1206
1207     eval { run_command([$binary, '--version'], outfunc => $code); };
1208     warn $@ if $@;
1209
1210     return $kvm_user_version->{$binary};
1211
1212 }
1213 my sub extract_version {
1214     my ($machine_type, $version) = @_;
1215     $version = kvm_user_version() if !defined($version);
1216     PVE::QemuServer::Machine::extract_version($machine_type, $version)
1217 }
1218
1219 sub kernel_has_vhost_net {
1220     return -c '/dev/vhost-net';
1221 }
1222
1223 sub option_exists {
1224     my $key = shift;
1225     return defined($confdesc->{$key});
1226 }
1227
1228 my $cdrom_path;
1229 sub get_cdrom_path {
1230
1231     return  $cdrom_path if $cdrom_path;
1232
1233     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1234     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1235     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1236 }
1237
1238 sub get_iso_path {
1239     my ($storecfg, $vmid, $cdrom) = @_;
1240
1241     if ($cdrom eq 'cdrom') {
1242         return get_cdrom_path();
1243     } elsif ($cdrom eq 'none') {
1244         return '';
1245     } elsif ($cdrom =~ m|^/|) {
1246         return $cdrom;
1247     } else {
1248         return PVE::Storage::path($storecfg, $cdrom);
1249     }
1250 }
1251
1252 # try to convert old style file names to volume IDs
1253 sub filename_to_volume_id {
1254     my ($vmid, $file, $media) = @_;
1255
1256      if (!($file eq 'none' || $file eq 'cdrom' ||
1257           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1258
1259         return if $file =~ m|/|;
1260
1261         if ($media && $media eq 'cdrom') {
1262             $file = "local:iso/$file";
1263         } else {
1264             $file = "local:$vmid/$file";
1265         }
1266     }
1267
1268     return $file;
1269 }
1270
1271 sub verify_media_type {
1272     my ($opt, $vtype, $media) = @_;
1273
1274     return if !$media;
1275
1276     my $etype;
1277     if ($media eq 'disk') {
1278         $etype = 'images';
1279     } elsif ($media eq 'cdrom') {
1280         $etype = 'iso';
1281     } else {
1282         die "internal error";
1283     }
1284
1285     return if ($vtype eq $etype);
1286
1287     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1288 }
1289
1290 sub cleanup_drive_path {
1291     my ($opt, $storecfg, $drive) = @_;
1292
1293     # try to convert filesystem paths to volume IDs
1294
1295     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1296         ($drive->{file} !~ m|^/dev/.+|) &&
1297         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1298         ($drive->{file} !~ m/^\d+$/)) {
1299         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1300         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1301             if !$vtype;
1302         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1303         verify_media_type($opt, $vtype, $drive->{media});
1304         $drive->{file} = $volid;
1305     }
1306
1307     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1308 }
1309
1310 sub parse_hotplug_features {
1311     my ($data) = @_;
1312
1313     my $res = {};
1314
1315     return $res if $data eq '0';
1316
1317     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1318
1319     foreach my $feature (PVE::Tools::split_list($data)) {
1320         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1321             $res->{$1} = 1;
1322         } else {
1323             die "invalid hotplug feature '$feature'\n";
1324         }
1325     }
1326     return $res;
1327 }
1328
1329 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1330 sub pve_verify_hotplug_features {
1331     my ($value, $noerr) = @_;
1332
1333     return $value if parse_hotplug_features($value);
1334
1335     return if $noerr;
1336
1337     die "unable to parse hotplug option\n";
1338 }
1339
1340 sub scsi_inquiry {
1341     my($fh, $noerr) = @_;
1342
1343     my $SG_IO = 0x2285;
1344     my $SG_GET_VERSION_NUM = 0x2282;
1345
1346     my $versionbuf = "\x00" x 8;
1347     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1348     if (!$ret) {
1349         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1350         return;
1351     }
1352     my $version = unpack("I", $versionbuf);
1353     if ($version < 30000) {
1354         die "scsi generic interface too old\n"  if !$noerr;
1355         return;
1356     }
1357
1358     my $buf = "\x00" x 36;
1359     my $sensebuf = "\x00" x 8;
1360     my $cmd = pack("C x3 C x1", 0x12, 36);
1361
1362     # see /usr/include/scsi/sg.h
1363     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1364
1365     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1366                       length($sensebuf), 0, length($buf), $buf,
1367                       $cmd, $sensebuf, 6000);
1368
1369     $ret = ioctl($fh, $SG_IO, $packet);
1370     if (!$ret) {
1371         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1372         return;
1373     }
1374
1375     my @res = unpack($sg_io_hdr_t, $packet);
1376     if ($res[17] || $res[18]) {
1377         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1378         return;
1379     }
1380
1381     my $res = {};
1382     (my $byte0, my $byte1, $res->{vendor},
1383      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1384
1385     $res->{removable} = $byte1 & 128 ? 1 : 0;
1386     $res->{type} = $byte0 & 31;
1387
1388     return $res;
1389 }
1390
1391 sub path_is_scsi {
1392     my ($path) = @_;
1393
1394     my $fh = IO::File->new("+<$path") || return;
1395     my $res = scsi_inquiry($fh, 1);
1396     close($fh);
1397
1398     return $res;
1399 }
1400
1401 sub print_tabletdevice_full {
1402     my ($conf, $arch) = @_;
1403
1404     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1405
1406     # we use uhci for old VMs because tablet driver was buggy in older qemu
1407     my $usbbus;
1408     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1409         $usbbus = 'ehci';
1410     } else {
1411         $usbbus = 'uhci';
1412     }
1413
1414     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1415 }
1416
1417 sub print_keyboarddevice_full {
1418     my ($conf, $arch, $machine) = @_;
1419
1420     return if $arch ne 'aarch64';
1421
1422     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1423 }
1424
1425 my sub get_drive_id {
1426     my ($drive) = @_;
1427     return "$drive->{interface}$drive->{index}";
1428 }
1429
1430 sub print_drivedevice_full {
1431     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1432
1433     my $device = '';
1434     my $maxdev = 0;
1435
1436     my $drive_id = get_drive_id($drive);
1437     if ($drive->{interface} eq 'virtio') {
1438         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1439         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1440         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1441     } elsif ($drive->{interface} eq 'scsi') {
1442
1443         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1444         my $unit = $drive->{index} % $maxdev;
1445         my $devicetype = 'hd';
1446         my $path = '';
1447         if (drive_is_cdrom($drive)) {
1448             $devicetype = 'cd';
1449         } else {
1450             if ($drive->{file} =~ m|^/|) {
1451                 $path = $drive->{file};
1452                 if (my $info = path_is_scsi($path)) {
1453                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1454                         $devicetype = 'block';
1455                     } elsif ($info->{type} == 1) { # tape
1456                         $devicetype = 'generic';
1457                     }
1458                 }
1459             } else {
1460                  $path = PVE::Storage::path($storecfg, $drive->{file});
1461             }
1462
1463             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1464             my $version = extract_version($machine_type, kvm_user_version());
1465             if ($path =~ m/^iscsi\:\/\// &&
1466                !min_version($version, 4, 1)) {
1467                 $devicetype = 'generic';
1468             }
1469         }
1470
1471         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1472             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1473         } else {
1474             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1475                 .",lun=$drive->{index}";
1476         }
1477         $device .= ",drive=drive-$drive_id,id=$drive_id";
1478
1479         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1480             $device .= ",rotation_rate=1";
1481         }
1482         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1483
1484     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1485         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1486         my $controller = int($drive->{index} / $maxdev);
1487         my $unit = $drive->{index} % $maxdev;
1488         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1489
1490         $device = "ide-$devicetype";
1491         if ($drive->{interface} eq 'ide') {
1492             $device .= ",bus=ide.$controller,unit=$unit";
1493         } else {
1494             $device .= ",bus=ahci$controller.$unit";
1495         }
1496         $device .= ",drive=drive-$drive_id,id=$drive_id";
1497
1498         if ($devicetype eq 'hd') {
1499             if (my $model = $drive->{model}) {
1500                 $model = URI::Escape::uri_unescape($model);
1501                 $device .= ",model=$model";
1502             }
1503             if ($drive->{ssd}) {
1504                 $device .= ",rotation_rate=1";
1505             }
1506         }
1507         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1508     } elsif ($drive->{interface} eq 'usb') {
1509         die "implement me";
1510         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1511     } else {
1512         die "unsupported interface type";
1513     }
1514
1515     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1516
1517     if (my $serial = $drive->{serial}) {
1518         $serial = URI::Escape::uri_unescape($serial);
1519         $device .= ",serial=$serial";
1520     }
1521
1522
1523     return $device;
1524 }
1525
1526 sub get_initiator_name {
1527     my $initiator;
1528
1529     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1530     while (defined(my $line = <$fh>)) {
1531         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1532         $initiator = $1;
1533         last;
1534     }
1535     $fh->close();
1536
1537     return $initiator;
1538 }
1539
1540 sub print_drive_commandline_full {
1541     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1542
1543     my $path;
1544     my $volid = $drive->{file};
1545     my $format = $drive->{format};
1546     my $drive_id = get_drive_id($drive);
1547
1548     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1549     my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1550
1551     if (drive_is_cdrom($drive)) {
1552         $path = get_iso_path($storecfg, $vmid, $volid);
1553         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1554     } else {
1555         if ($storeid) {
1556             $path = PVE::Storage::path($storecfg, $volid);
1557             $format //= qemu_img_format($scfg, $volname);
1558         } else {
1559             $path = $volid;
1560             $format //= "raw";
1561         }
1562    }
1563
1564    my $is_rbd = $path =~ m/^rbd:/;
1565
1566     my $opts = '';
1567     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1568     foreach my $o (@qemu_drive_options) {
1569         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1570     }
1571
1572     # snapshot only accepts on|off
1573     if (defined($drive->{snapshot})) {
1574         my $v = $drive->{snapshot} ? 'on' : 'off';
1575         $opts .= ",snapshot=$v";
1576     }
1577
1578     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1579         my ($dir, $qmpname) = @$type;
1580         if (my $v = $drive->{"mbps$dir"}) {
1581             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1582         }
1583         if (my $v = $drive->{"mbps${dir}_max"}) {
1584             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1585         }
1586         if (my $v = $drive->{"bps${dir}_max_length"}) {
1587             $opts .= ",throttling.bps$qmpname-max-length=$v";
1588         }
1589         if (my $v = $drive->{"iops${dir}"}) {
1590             $opts .= ",throttling.iops$qmpname=$v";
1591         }
1592         if (my $v = $drive->{"iops${dir}_max"}) {
1593             $opts .= ",throttling.iops$qmpname-max=$v";
1594         }
1595         if (my $v = $drive->{"iops${dir}_max_length"}) {
1596             $opts .= ",throttling.iops$qmpname-max-length=$v";
1597         }
1598     }
1599
1600     if ($pbs_name) {
1601         $format = "rbd" if $is_rbd;
1602         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1603             if !$format;
1604         $opts .= ",format=alloc-track,file.driver=$format";
1605     } elsif ($format) {
1606         $opts .= ",format=$format";
1607     }
1608
1609     my $cache_direct = 0;
1610
1611     if (my $cache = $drive->{cache}) {
1612         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1613     } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1614         $opts .= ",cache=none";
1615         $cache_direct = 1;
1616     }
1617
1618     # io_uring with cache mode writeback or writethrough on krbd will hang...
1619     my $rbd_no_io_uring = $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1620
1621     # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1622     # sometimes, just plain disable...
1623     my $lvm_no_io_uring = $scfg && $scfg->{type} eq 'lvm';
1624
1625     if (!$drive->{aio}) {
1626         if ($io_uring && !$rbd_no_io_uring && !$lvm_no_io_uring) {
1627             # io_uring supports all cache modes
1628             $opts .= ",aio=io_uring";
1629         } else {
1630             # aio native works only with O_DIRECT
1631             if($cache_direct) {
1632                 $opts .= ",aio=native";
1633             } else {
1634                 $opts .= ",aio=threads";
1635             }
1636         }
1637     }
1638
1639     if (!drive_is_cdrom($drive)) {
1640         my $detectzeroes;
1641         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1642             $detectzeroes = 'off';
1643         } elsif ($drive->{discard}) {
1644             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1645         } else {
1646             # This used to be our default with discard not being specified:
1647             $detectzeroes = 'on';
1648         }
1649
1650         # note: 'detect-zeroes' works per blockdev and we want it to persist
1651         # after the alloc-track is removed, so put it on 'file' directly
1652         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1653         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1654     }
1655
1656     if ($pbs_name) {
1657         $opts .= ",backing=$pbs_name";
1658         $opts .= ",auto-remove=on";
1659     }
1660
1661     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1662     my $file_param = "file";
1663     if ($pbs_name) {
1664         # non-rbd drivers require the underlying file to be a seperate block
1665         # node, so add a second .file indirection
1666         $file_param .= ".file" if !$is_rbd;
1667         $file_param .= ".filename";
1668     }
1669     my $pathinfo = $path ? "$file_param=$path," : '';
1670
1671     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1672 }
1673
1674 sub print_pbs_blockdev {
1675     my ($pbs_conf, $pbs_name) = @_;
1676     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1677     $blockdev .= ",repository=$pbs_conf->{repository}";
1678     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1679     $blockdev .= ",archive=$pbs_conf->{archive}";
1680     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1681     return $blockdev;
1682 }
1683
1684 sub print_netdevice_full {
1685     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1686
1687     my $device = $net->{model};
1688     if ($net->{model} eq 'virtio') {
1689          $device = 'virtio-net-pci';
1690      };
1691
1692     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1693     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1694     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1695         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1696         # and out of each queue plus one config interrupt and control vector queue
1697         my $vectors = $net->{queues} * 2 + 2;
1698         $tmpstr .= ",vectors=$vectors,mq=on";
1699     }
1700     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1701
1702     if (my $mtu = $net->{mtu}) {
1703         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1704             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1705             if ($mtu == 1) {
1706                  $mtu = $bridge_mtu;
1707             } elsif ($mtu < 576) {
1708                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1709             } elsif ($mtu > $bridge_mtu) {
1710                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1711             }
1712             $tmpstr .= ",host_mtu=$mtu";
1713         } else {
1714             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1715         }
1716     }
1717
1718     if ($use_old_bios_files) {
1719         my $romfile;
1720         if ($device eq 'virtio-net-pci') {
1721             $romfile = 'pxe-virtio.rom';
1722         } elsif ($device eq 'e1000') {
1723             $romfile = 'pxe-e1000.rom';
1724         } elsif ($device eq 'e1000e') {
1725             $romfile = 'pxe-e1000e.rom';
1726         } elsif ($device eq 'ne2k') {
1727             $romfile = 'pxe-ne2k_pci.rom';
1728         } elsif ($device eq 'pcnet') {
1729             $romfile = 'pxe-pcnet.rom';
1730         } elsif ($device eq 'rtl8139') {
1731             $romfile = 'pxe-rtl8139.rom';
1732         }
1733         $tmpstr .= ",romfile=$romfile" if $romfile;
1734     }
1735
1736     return $tmpstr;
1737 }
1738
1739 sub print_netdev_full {
1740     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1741
1742     my $i = '';
1743     if ($netid =~ m/^net(\d+)$/) {
1744         $i = int($1);
1745     }
1746
1747     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1748
1749     my $ifname = "tap${vmid}i$i";
1750
1751     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1752     die "interface name '$ifname' is too long (max 15 character)\n"
1753         if length($ifname) >= 16;
1754
1755     my $vhostparam = '';
1756     if (is_native($arch)) {
1757         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1758     }
1759
1760     my $vmname = $conf->{name} || "vm$vmid";
1761
1762     my $netdev = "";
1763     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1764
1765     if ($net->{bridge}) {
1766         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1767             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1768     } else {
1769         $netdev = "type=user,id=$netid,hostname=$vmname";
1770     }
1771
1772     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1773
1774     return $netdev;
1775 }
1776
1777 my $vga_map = {
1778     'cirrus' => 'cirrus-vga',
1779     'std' => 'VGA',
1780     'vmware' => 'vmware-svga',
1781     'virtio' => 'virtio-vga',
1782 };
1783
1784 sub print_vga_device {
1785     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1786
1787     my $type = $vga_map->{$vga->{type}};
1788     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1789         $type = 'virtio-gpu';
1790     }
1791     my $vgamem_mb = $vga->{memory};
1792
1793     my $max_outputs = '';
1794     if ($qxlnum) {
1795         $type = $id ? 'qxl' : 'qxl-vga';
1796
1797         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1798             # set max outputs so linux can have up to 4 qxl displays with one device
1799             if (min_version($machine_version, 4, 1)) {
1800                 $max_outputs = ",max_outputs=4";
1801             }
1802         }
1803     }
1804
1805     die "no devicetype for $vga->{type}\n" if !$type;
1806
1807     my $memory = "";
1808     if ($vgamem_mb) {
1809         if ($vga->{type} eq 'virtio') {
1810             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1811             $memory = ",max_hostmem=$bytes";
1812         } elsif ($qxlnum) {
1813             # from https://www.spice-space.org/multiple-monitors.html
1814             $memory = ",vgamem_mb=$vga->{memory}";
1815             my $ram = $vgamem_mb * 4;
1816             my $vram = $vgamem_mb * 2;
1817             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1818         } else {
1819             $memory = ",vgamem_mb=$vga->{memory}";
1820         }
1821     } elsif ($qxlnum && $id) {
1822         $memory = ",ram_size=67108864,vram_size=33554432";
1823     }
1824
1825     my $edidoff = "";
1826     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1827         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1828     }
1829
1830     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1831     my $vgaid = "vga" . ($id // '');
1832     my $pciaddr;
1833
1834     if ($q35 && $vgaid eq 'vga') {
1835         # the first display uses pcie.0 bus on q35 machines
1836         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1837     } else {
1838         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1839     }
1840
1841     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1842 }
1843
1844 sub parse_number_sets {
1845     my ($set) = @_;
1846     my $res = [];
1847     foreach my $part (split(/;/, $set)) {
1848         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1849             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1850             push @$res, [ $1, $2 ];
1851         } else {
1852             die "invalid range: $part\n";
1853         }
1854     }
1855     return $res;
1856 }
1857
1858 sub parse_numa {
1859     my ($data) = @_;
1860
1861     my $res = parse_property_string($numa_fmt, $data);
1862     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1863     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1864     return $res;
1865 }
1866
1867 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1868 sub parse_net {
1869     my ($data) = @_;
1870
1871     my $res = eval { parse_property_string($net_fmt, $data) };
1872     if ($@) {
1873         warn $@;
1874         return;
1875     }
1876     if (!defined($res->{macaddr})) {
1877         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1878         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1879     }
1880     return $res;
1881 }
1882
1883 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1884 sub parse_ipconfig {
1885     my ($data) = @_;
1886
1887     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1888     if ($@) {
1889         warn $@;
1890         return;
1891     }
1892
1893     if ($res->{gw} && !$res->{ip}) {
1894         warn 'gateway specified without specifying an IP address';
1895         return;
1896     }
1897     if ($res->{gw6} && !$res->{ip6}) {
1898         warn 'IPv6 gateway specified without specifying an IPv6 address';
1899         return;
1900     }
1901     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1902         warn 'gateway specified together with DHCP';
1903         return;
1904     }
1905     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1906         # gw6 + auto/dhcp
1907         warn "IPv6 gateway specified together with $res->{ip6} address";
1908         return;
1909     }
1910
1911     if (!$res->{ip} && !$res->{ip6}) {
1912         return { ip => 'dhcp', ip6 => 'dhcp' };
1913     }
1914
1915     return $res;
1916 }
1917
1918 sub print_net {
1919     my $net = shift;
1920
1921     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1922 }
1923
1924 sub add_random_macs {
1925     my ($settings) = @_;
1926
1927     foreach my $opt (keys %$settings) {
1928         next if $opt !~ m/^net(\d+)$/;
1929         my $net = parse_net($settings->{$opt});
1930         next if !$net;
1931         $settings->{$opt} = print_net($net);
1932     }
1933 }
1934
1935 sub vm_is_volid_owner {
1936     my ($storecfg, $vmid, $volid) = @_;
1937
1938     if ($volid !~  m|^/|) {
1939         my ($path, $owner);
1940         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1941         if ($owner && ($owner == $vmid)) {
1942             return 1;
1943         }
1944     }
1945
1946     return;
1947 }
1948
1949 sub vmconfig_register_unused_drive {
1950     my ($storecfg, $vmid, $conf, $drive) = @_;
1951
1952     if (drive_is_cloudinit($drive)) {
1953         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1954         warn $@ if $@;
1955     } elsif (!drive_is_cdrom($drive)) {
1956         my $volid = $drive->{file};
1957         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1958             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1959         }
1960     }
1961 }
1962
1963 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1964 my $smbios1_fmt = {
1965     uuid => {
1966         type => 'string',
1967         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1968         format_description => 'UUID',
1969         description => "Set SMBIOS1 UUID.",
1970         optional => 1,
1971     },
1972     version => {
1973         type => 'string',
1974         pattern => '[A-Za-z0-9+\/]+={0,2}',
1975         format_description => 'Base64 encoded string',
1976         description => "Set SMBIOS1 version.",
1977         optional => 1,
1978     },
1979     serial => {
1980         type => 'string',
1981         pattern => '[A-Za-z0-9+\/]+={0,2}',
1982         format_description => 'Base64 encoded string',
1983         description => "Set SMBIOS1 serial number.",
1984         optional => 1,
1985     },
1986     manufacturer => {
1987         type => 'string',
1988         pattern => '[A-Za-z0-9+\/]+={0,2}',
1989         format_description => 'Base64 encoded string',
1990         description => "Set SMBIOS1 manufacturer.",
1991         optional => 1,
1992     },
1993     product => {
1994         type => 'string',
1995         pattern => '[A-Za-z0-9+\/]+={0,2}',
1996         format_description => 'Base64 encoded string',
1997         description => "Set SMBIOS1 product ID.",
1998         optional => 1,
1999     },
2000     sku => {
2001         type => 'string',
2002         pattern => '[A-Za-z0-9+\/]+={0,2}',
2003         format_description => 'Base64 encoded string',
2004         description => "Set SMBIOS1 SKU string.",
2005         optional => 1,
2006     },
2007     family => {
2008         type => 'string',
2009         pattern => '[A-Za-z0-9+\/]+={0,2}',
2010         format_description => 'Base64 encoded string',
2011         description => "Set SMBIOS1 family string.",
2012         optional => 1,
2013     },
2014     base64 => {
2015         type => 'boolean',
2016         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2017         optional => 1,
2018     },
2019 };
2020
2021 sub parse_smbios1 {
2022     my ($data) = @_;
2023
2024     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2025     warn $@ if $@;
2026     return $res;
2027 }
2028
2029 sub print_smbios1 {
2030     my ($smbios1) = @_;
2031     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2032 }
2033
2034 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2035
2036 sub parse_watchdog {
2037     my ($value) = @_;
2038
2039     return if !$value;
2040
2041     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2042     warn $@ if $@;
2043     return $res;
2044 }
2045
2046 sub parse_guest_agent {
2047     my ($conf) = @_;
2048
2049     return {} if !defined($conf->{agent});
2050
2051     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2052     warn $@ if $@;
2053
2054     # if the agent is disabled ignore the other potentially set properties
2055     return {} if !$res->{enabled};
2056     return $res;
2057 }
2058
2059 sub get_qga_key {
2060     my ($conf, $key) = @_;
2061     return undef if !defined($conf->{agent});
2062
2063     my $agent = parse_guest_agent($conf);
2064     return $agent->{$key};
2065 }
2066
2067 sub parse_vga {
2068     my ($value) = @_;
2069
2070     return {} if !$value;
2071     my $res = eval { parse_property_string($vga_fmt, $value) };
2072     warn $@ if $@;
2073     return $res;
2074 }
2075
2076 sub parse_rng {
2077     my ($value) = @_;
2078
2079     return if !$value;
2080
2081     my $res = eval { parse_property_string($rng_fmt, $value) };
2082     warn $@ if $@;
2083     return $res;
2084 }
2085
2086 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2087 sub verify_usb_device {
2088     my ($value, $noerr) = @_;
2089
2090     return $value if parse_usb_device($value);
2091
2092     return if $noerr;
2093
2094     die "unable to parse usb device\n";
2095 }
2096
2097 # add JSON properties for create and set function
2098 sub json_config_properties {
2099     my $prop = shift;
2100
2101     foreach my $opt (keys %$confdesc) {
2102         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' ||
2103             $opt eq 'runningmachine' || $opt eq 'runningcpu';
2104         $prop->{$opt} = $confdesc->{$opt};
2105     }
2106
2107     return $prop;
2108 }
2109
2110 # return copy of $confdesc_cloudinit to generate documentation
2111 sub cloudinit_config_properties {
2112
2113     return dclone($confdesc_cloudinit);
2114 }
2115
2116 sub check_type {
2117     my ($key, $value) = @_;
2118
2119     die "unknown setting '$key'\n" if !$confdesc->{$key};
2120
2121     my $type = $confdesc->{$key}->{type};
2122
2123     if (!defined($value)) {
2124         die "got undefined value\n";
2125     }
2126
2127     if ($value =~ m/[\n\r]/) {
2128         die "property contains a line feed\n";
2129     }
2130
2131     if ($type eq 'boolean') {
2132         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2133         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2134         die "type check ('boolean') failed - got '$value'\n";
2135     } elsif ($type eq 'integer') {
2136         return int($1) if $value =~ m/^(\d+)$/;
2137         die "type check ('integer') failed - got '$value'\n";
2138     } elsif ($type eq 'number') {
2139         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2140         die "type check ('number') failed - got '$value'\n";
2141     } elsif ($type eq 'string') {
2142         if (my $fmt = $confdesc->{$key}->{format}) {
2143             PVE::JSONSchema::check_format($fmt, $value);
2144             return $value;
2145         }
2146         $value =~ s/^\"(.*)\"$/$1/;
2147         return $value;
2148     } else {
2149         die "internal error"
2150     }
2151 }
2152
2153 sub destroy_vm {
2154     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2155
2156     my $conf = PVE::QemuConfig->load_config($vmid);
2157
2158     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2159
2160     if ($conf->{template}) {
2161         # check if any base image is still used by a linked clone
2162         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2163                 my ($ds, $drive) = @_;
2164                 return if drive_is_cdrom($drive);
2165
2166                 my $volid = $drive->{file};
2167                 return if !$volid || $volid =~ m|^/|;
2168
2169                 die "base volume '$volid' is still in use by linked cloned\n"
2170                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2171
2172         });
2173     }
2174
2175     my $volids = {};
2176     my $remove_owned_drive = sub {
2177         my ($ds, $drive) = @_;
2178         return if drive_is_cdrom($drive, 1);
2179
2180         my $volid = $drive->{file};
2181         return if !$volid || $volid =~ m|^/|;
2182         return if $volids->{$volid};
2183
2184         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2185         return if !$path || !$owner || ($owner != $vmid);
2186
2187         $volids->{$volid} = 1;
2188         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2189         warn "Could not remove disk '$volid', check manually: $@" if $@;
2190     };
2191
2192     # only remove disks owned by this VM (referenced in the config)
2193     my $include_opts = {
2194         include_unused => 1,
2195         extra_keys => ['vmstate'],
2196     };
2197     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2198
2199     for my $snap (values %{$conf->{snapshots}}) {
2200         next if !defined($snap->{vmstate});
2201         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2202         next if !defined($drive);
2203         $remove_owned_drive->('vmstate', $drive);
2204     }
2205
2206     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive);
2207
2208     if ($purge_unreferenced) { # also remove unreferenced disk
2209         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2210         PVE::Storage::foreach_volid($vmdisks, sub {
2211             my ($volid, $sid, $volname, $d) = @_;
2212             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2213             warn $@ if $@;
2214         });
2215     }
2216
2217     if (defined $replacement_conf) {
2218         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2219     } else {
2220         PVE::QemuConfig->destroy_config($vmid);
2221     }
2222 }
2223
2224 sub parse_vm_config {
2225     my ($filename, $raw) = @_;
2226
2227     return if !defined($raw);
2228
2229     my $res = {
2230         digest => Digest::SHA::sha1_hex($raw),
2231         snapshots => {},
2232         pending => {},
2233     };
2234
2235     $filename =~ m|/qemu-server/(\d+)\.conf$|
2236         || die "got strange filename '$filename'";
2237
2238     my $vmid = $1;
2239
2240     my $conf = $res;
2241     my $descr;
2242     my $section = '';
2243
2244     my @lines = split(/\n/, $raw);
2245     foreach my $line (@lines) {
2246         next if $line =~ m/^\s*$/;
2247
2248         if ($line =~ m/^\[PENDING\]\s*$/i) {
2249             $section = 'pending';
2250             if (defined($descr)) {
2251                 $descr =~ s/\s+$//;
2252                 $conf->{description} = $descr;
2253             }
2254             $descr = undef;
2255             $conf = $res->{$section} = {};
2256             next;
2257
2258         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2259             $section = $1;
2260             if (defined($descr)) {
2261                 $descr =~ s/\s+$//;
2262                 $conf->{description} = $descr;
2263             }
2264             $descr = undef;
2265             $conf = $res->{snapshots}->{$section} = {};
2266             next;
2267         }
2268
2269         if ($line =~ m/^\#(.*)\s*$/) {
2270             $descr = '' if !defined($descr);
2271             $descr .= PVE::Tools::decode_text($1) . "\n";
2272             next;
2273         }
2274
2275         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2276             $descr = '' if !defined($descr);
2277             $descr .= PVE::Tools::decode_text($2);
2278         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2279             $conf->{snapstate} = $1;
2280         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2281             my $key = $1;
2282             my $value = $2;
2283             $conf->{$key} = $value;
2284         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2285             my $value = $1;
2286             if ($section eq 'pending') {
2287                 $conf->{delete} = $value; # we parse this later
2288             } else {
2289                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2290             }
2291         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2292             my $key = $1;
2293             my $value = $2;
2294             eval { $value = check_type($key, $value); };
2295             if ($@) {
2296                 warn "vm $vmid - unable to parse value of '$key' - $@";
2297             } else {
2298                 $key = 'ide2' if $key eq 'cdrom';
2299                 my $fmt = $confdesc->{$key}->{format};
2300                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2301                     my $v = parse_drive($key, $value);
2302                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2303                         $v->{file} = $volid;
2304                         $value = print_drive($v);
2305                     } else {
2306                         warn "vm $vmid - unable to parse value of '$key'\n";
2307                         next;
2308                     }
2309                 }
2310
2311                 $conf->{$key} = $value;
2312             }
2313         } else {
2314             warn "vm $vmid - unable to parse config: $line\n";
2315         }
2316     }
2317
2318     if (defined($descr)) {
2319         $descr =~ s/\s+$//;
2320         $conf->{description} = $descr;
2321     }
2322     delete $res->{snapstate}; # just to be sure
2323
2324     return $res;
2325 }
2326
2327 sub write_vm_config {
2328     my ($filename, $conf) = @_;
2329
2330     delete $conf->{snapstate}; # just to be sure
2331
2332     if ($conf->{cdrom}) {
2333         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2334         $conf->{ide2} = $conf->{cdrom};
2335         delete $conf->{cdrom};
2336     }
2337
2338     # we do not use 'smp' any longer
2339     if ($conf->{sockets}) {
2340         delete $conf->{smp};
2341     } elsif ($conf->{smp}) {
2342         $conf->{sockets} = $conf->{smp};
2343         delete $conf->{cores};
2344         delete $conf->{smp};
2345     }
2346
2347     my $used_volids = {};
2348
2349     my $cleanup_config = sub {
2350         my ($cref, $pending, $snapname) = @_;
2351
2352         foreach my $key (keys %$cref) {
2353             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2354                 $key eq 'snapstate' || $key eq 'pending';
2355             my $value = $cref->{$key};
2356             if ($key eq 'delete') {
2357                 die "propertry 'delete' is only allowed in [PENDING]\n"
2358                     if !$pending;
2359                 # fixme: check syntax?
2360                 next;
2361             }
2362             eval { $value = check_type($key, $value); };
2363             die "unable to parse value of '$key' - $@" if $@;
2364
2365             $cref->{$key} = $value;
2366
2367             if (!$snapname && is_valid_drivename($key)) {
2368                 my $drive = parse_drive($key, $value);
2369                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2370             }
2371         }
2372     };
2373
2374     &$cleanup_config($conf);
2375
2376     &$cleanup_config($conf->{pending}, 1);
2377
2378     foreach my $snapname (keys %{$conf->{snapshots}}) {
2379         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2380         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2381     }
2382
2383     # remove 'unusedX' settings if we re-add a volume
2384     foreach my $key (keys %$conf) {
2385         my $value = $conf->{$key};
2386         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2387             delete $conf->{$key};
2388         }
2389     }
2390
2391     my $generate_raw_config = sub {
2392         my ($conf, $pending) = @_;
2393
2394         my $raw = '';
2395
2396         # add description as comment to top of file
2397         if (defined(my $descr = $conf->{description})) {
2398             if ($descr) {
2399                 foreach my $cl (split(/\n/, $descr)) {
2400                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2401                 }
2402             } else {
2403                 $raw .= "#\n" if $pending;
2404             }
2405         }
2406
2407         foreach my $key (sort keys %$conf) {
2408             next if $key =~ /^(digest|description|pending|snapshots)$/;
2409             $raw .= "$key: $conf->{$key}\n";
2410         }
2411         return $raw;
2412     };
2413
2414     my $raw = &$generate_raw_config($conf);
2415
2416     if (scalar(keys %{$conf->{pending}})){
2417         $raw .= "\n[PENDING]\n";
2418         $raw .= &$generate_raw_config($conf->{pending}, 1);
2419     }
2420
2421     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2422         $raw .= "\n[$snapname]\n";
2423         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2424     }
2425
2426     return $raw;
2427 }
2428
2429 sub load_defaults {
2430
2431     my $res = {};
2432
2433     # we use static defaults from our JSON schema configuration
2434     foreach my $key (keys %$confdesc) {
2435         if (defined(my $default = $confdesc->{$key}->{default})) {
2436             $res->{$key} = $default;
2437         }
2438     }
2439
2440     return $res;
2441 }
2442
2443 sub config_list {
2444     my $vmlist = PVE::Cluster::get_vmlist();
2445     my $res = {};
2446     return $res if !$vmlist || !$vmlist->{ids};
2447     my $ids = $vmlist->{ids};
2448     my $nodename = nodename();
2449
2450     foreach my $vmid (keys %$ids) {
2451         my $d = $ids->{$vmid};
2452         next if !$d->{node} || $d->{node} ne $nodename;
2453         next if !$d->{type} || $d->{type} ne 'qemu';
2454         $res->{$vmid}->{exists} = 1;
2455     }
2456     return $res;
2457 }
2458
2459 # test if VM uses local resources (to prevent migration)
2460 sub check_local_resources {
2461     my ($conf, $noerr) = @_;
2462
2463     my @loc_res = ();
2464
2465     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2466     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2467
2468     push @loc_res, "ivshmem" if $conf->{ivshmem};
2469
2470     foreach my $k (keys %$conf) {
2471         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2472         # sockets are safe: they will recreated be on the target side post-migrate
2473         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2474         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2475     }
2476
2477     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2478
2479     return \@loc_res;
2480 }
2481
2482 # check if used storages are available on all nodes (use by migrate)
2483 sub check_storage_availability {
2484     my ($storecfg, $conf, $node) = @_;
2485
2486     PVE::QemuConfig->foreach_volume($conf, sub {
2487         my ($ds, $drive) = @_;
2488
2489         my $volid = $drive->{file};
2490         return if !$volid;
2491
2492         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2493         return if !$sid;
2494
2495         # check if storage is available on both nodes
2496         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2497         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2498
2499         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2500
2501         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2502             if !$scfg->{content}->{$vtype};
2503    });
2504 }
2505
2506 # list nodes where all VM images are available (used by has_feature API)
2507 sub shared_nodes {
2508     my ($conf, $storecfg) = @_;
2509
2510     my $nodelist = PVE::Cluster::get_nodelist();
2511     my $nodehash = { map { $_ => 1 } @$nodelist };
2512     my $nodename = nodename();
2513
2514     PVE::QemuConfig->foreach_volume($conf, sub {
2515         my ($ds, $drive) = @_;
2516
2517         my $volid = $drive->{file};
2518         return if !$volid;
2519
2520         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2521         if ($storeid) {
2522             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2523             if ($scfg->{disable}) {
2524                 $nodehash = {};
2525             } elsif (my $avail = $scfg->{nodes}) {
2526                 foreach my $node (keys %$nodehash) {
2527                     delete $nodehash->{$node} if !$avail->{$node};
2528                 }
2529             } elsif (!$scfg->{shared}) {
2530                 foreach my $node (keys %$nodehash) {
2531                     delete $nodehash->{$node} if $node ne $nodename
2532                 }
2533             }
2534         }
2535     });
2536
2537     return $nodehash
2538 }
2539
2540 sub check_local_storage_availability {
2541     my ($conf, $storecfg) = @_;
2542
2543     my $nodelist = PVE::Cluster::get_nodelist();
2544     my $nodehash = { map { $_ => {} } @$nodelist };
2545
2546     PVE::QemuConfig->foreach_volume($conf, sub {
2547         my ($ds, $drive) = @_;
2548
2549         my $volid = $drive->{file};
2550         return if !$volid;
2551
2552         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2553         if ($storeid) {
2554             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2555
2556             if ($scfg->{disable}) {
2557                 foreach my $node (keys %$nodehash) {
2558                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2559                 }
2560             } elsif (my $avail = $scfg->{nodes}) {
2561                 foreach my $node (keys %$nodehash) {
2562                     if (!$avail->{$node}) {
2563                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2564                     }
2565                 }
2566             }
2567         }
2568     });
2569
2570     foreach my $node (values %$nodehash) {
2571         if (my $unavail = $node->{unavailable_storages}) {
2572             $node->{unavailable_storages} = [ sort keys %$unavail ];
2573         }
2574     }
2575
2576     return $nodehash
2577 }
2578
2579 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2580 sub check_running {
2581     my ($vmid, $nocheck, $node) = @_;
2582
2583     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2584     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2585 }
2586
2587 sub vzlist {
2588
2589     my $vzlist = config_list();
2590
2591     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2592
2593     while (defined(my $de = $fd->read)) {
2594         next if $de !~ m/^(\d+)\.pid$/;
2595         my $vmid = $1;
2596         next if !defined($vzlist->{$vmid});
2597         if (my $pid = check_running($vmid)) {
2598             $vzlist->{$vmid}->{pid} = $pid;
2599         }
2600     }
2601
2602     return $vzlist;
2603 }
2604
2605 our $vmstatus_return_properties = {
2606     vmid => get_standard_option('pve-vmid'),
2607     status => {
2608         description => "Qemu process status.",
2609         type => 'string',
2610         enum => ['stopped', 'running'],
2611     },
2612     maxmem => {
2613         description => "Maximum memory in bytes.",
2614         type => 'integer',
2615         optional => 1,
2616         renderer => 'bytes',
2617     },
2618     maxdisk => {
2619         description => "Root disk size in bytes.",
2620         type => 'integer',
2621         optional => 1,
2622         renderer => 'bytes',
2623     },
2624     name => {
2625         description => "VM name.",
2626         type => 'string',
2627         optional => 1,
2628     },
2629     qmpstatus => {
2630         description => "Qemu QMP agent status.",
2631         type => 'string',
2632         optional => 1,
2633     },
2634     pid => {
2635         description => "PID of running qemu process.",
2636         type => 'integer',
2637         optional => 1,
2638     },
2639     uptime => {
2640         description => "Uptime.",
2641         type => 'integer',
2642         optional => 1,
2643         renderer => 'duration',
2644     },
2645     cpus => {
2646         description => "Maximum usable CPUs.",
2647         type => 'number',
2648         optional => 1,
2649     },
2650     lock => {
2651         description => "The current config lock, if any.",
2652         type => 'string',
2653         optional => 1,
2654     },
2655     tags => {
2656         description  => "The current configured tags, if any",
2657         type => 'string',
2658         optional => 1,
2659     },
2660     'running-machine' => {
2661         description => "The currently running machine type (if running).",
2662         type => 'string',
2663         optional => 1,
2664     },
2665     'running-qemu' => {
2666         description => "The currently running QEMU version (if running).",
2667         type => 'string',
2668         optional => 1,
2669     },
2670 };
2671
2672 my $last_proc_pid_stat;
2673
2674 # get VM status information
2675 # This must be fast and should not block ($full == false)
2676 # We only query KVM using QMP if $full == true (this can be slow)
2677 sub vmstatus {
2678     my ($opt_vmid, $full) = @_;
2679
2680     my $res = {};
2681
2682     my $storecfg = PVE::Storage::config();
2683
2684     my $list = vzlist();
2685     my $defaults = load_defaults();
2686
2687     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2688
2689     my $cpucount = $cpuinfo->{cpus} || 1;
2690
2691     foreach my $vmid (keys %$list) {
2692         next if $opt_vmid && ($vmid ne $opt_vmid);
2693
2694         my $conf = PVE::QemuConfig->load_config($vmid);
2695
2696         my $d = { vmid => int($vmid) };
2697         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2698
2699         # fixme: better status?
2700         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2701
2702         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2703         if (defined($size)) {
2704             $d->{disk} = 0; # no info available
2705             $d->{maxdisk} = $size;
2706         } else {
2707             $d->{disk} = 0;
2708             $d->{maxdisk} = 0;
2709         }
2710
2711         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2712             * ($conf->{cores} || $defaults->{cores});
2713         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2714         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2715
2716         $d->{name} = $conf->{name} || "VM $vmid";
2717         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2718             : $defaults->{memory}*(1024*1024);
2719
2720         if ($conf->{balloon}) {
2721             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2722             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2723                 : $defaults->{shares};
2724         }
2725
2726         $d->{uptime} = 0;
2727         $d->{cpu} = 0;
2728         $d->{mem} = 0;
2729
2730         $d->{netout} = 0;
2731         $d->{netin} = 0;
2732
2733         $d->{diskread} = 0;
2734         $d->{diskwrite} = 0;
2735
2736         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2737
2738         $d->{serial} = 1 if conf_has_serial($conf);
2739         $d->{lock} = $conf->{lock} if $conf->{lock};
2740         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2741
2742         $res->{$vmid} = $d;
2743     }
2744
2745     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2746     foreach my $dev (keys %$netdev) {
2747         next if $dev !~ m/^tap([1-9]\d*)i/;
2748         my $vmid = $1;
2749         my $d = $res->{$vmid};
2750         next if !$d;
2751
2752         $d->{netout} += $netdev->{$dev}->{receive};
2753         $d->{netin} += $netdev->{$dev}->{transmit};
2754
2755         if ($full) {
2756             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2757             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2758         }
2759
2760     }
2761
2762     my $ctime = gettimeofday;
2763
2764     foreach my $vmid (keys %$list) {
2765
2766         my $d = $res->{$vmid};
2767         my $pid = $d->{pid};
2768         next if !$pid;
2769
2770         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2771         next if !$pstat; # not running
2772
2773         my $used = $pstat->{utime} + $pstat->{stime};
2774
2775         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2776
2777         if ($pstat->{vsize}) {
2778             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2779         }
2780
2781         my $old = $last_proc_pid_stat->{$pid};
2782         if (!$old) {
2783             $last_proc_pid_stat->{$pid} = {
2784                 time => $ctime,
2785                 used => $used,
2786                 cpu => 0,
2787             };
2788             next;
2789         }
2790
2791         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2792
2793         if ($dtime > 1000) {
2794             my $dutime = $used -  $old->{used};
2795
2796             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2797             $last_proc_pid_stat->{$pid} = {
2798                 time => $ctime,
2799                 used => $used,
2800                 cpu => $d->{cpu},
2801             };
2802         } else {
2803             $d->{cpu} = $old->{cpu};
2804         }
2805     }
2806
2807     return $res if !$full;
2808
2809     my $qmpclient = PVE::QMPClient->new();
2810
2811     my $ballooncb = sub {
2812         my ($vmid, $resp) = @_;
2813
2814         my $info = $resp->{'return'};
2815         return if !$info->{max_mem};
2816
2817         my $d = $res->{$vmid};
2818
2819         # use memory assigned to VM
2820         $d->{maxmem} = $info->{max_mem};
2821         $d->{balloon} = $info->{actual};
2822
2823         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2824             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2825             $d->{freemem} = $info->{free_mem};
2826         }
2827
2828         $d->{ballooninfo} = $info;
2829     };
2830
2831     my $blockstatscb = sub {
2832         my ($vmid, $resp) = @_;
2833         my $data = $resp->{'return'} || [];
2834         my $totalrdbytes = 0;
2835         my $totalwrbytes = 0;
2836
2837         for my $blockstat (@$data) {
2838             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2839             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2840
2841             $blockstat->{device} =~ s/drive-//;
2842             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2843         }
2844         $res->{$vmid}->{diskread} = $totalrdbytes;
2845         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2846     };
2847
2848     my $machinecb = sub {
2849         my ($vmid, $resp) = @_;
2850         my $data = $resp->{'return'} || [];
2851
2852         $res->{$vmid}->{'running-machine'} =
2853             PVE::QemuServer::Machine::current_from_query_machines($data);
2854     };
2855
2856     my $versioncb = sub {
2857         my ($vmid, $resp) = @_;
2858         my $data = $resp->{'return'} // {};
2859         my $version = 'unknown';
2860
2861         if (my $v = $data->{qemu}) {
2862             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
2863         }
2864
2865         $res->{$vmid}->{'running-qemu'} = $version;
2866     };
2867
2868     my $statuscb = sub {
2869         my ($vmid, $resp) = @_;
2870
2871         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2872         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
2873         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
2874         # this fails if ballon driver is not loaded, so this must be
2875         # the last commnand (following command are aborted if this fails).
2876         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2877
2878         my $status = 'unknown';
2879         if (!defined($status = $resp->{'return'}->{status})) {
2880             warn "unable to get VM status\n";
2881             return;
2882         }
2883
2884         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2885     };
2886
2887     foreach my $vmid (keys %$list) {
2888         next if $opt_vmid && ($vmid ne $opt_vmid);
2889         next if !$res->{$vmid}->{pid}; # not running
2890         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2891     }
2892
2893     $qmpclient->queue_execute(undef, 2);
2894
2895     foreach my $vmid (keys %$list) {
2896         next if $opt_vmid && ($vmid ne $opt_vmid);
2897         next if !$res->{$vmid}->{pid}; #not running
2898
2899         # we can't use the $qmpclient since it might have already aborted on
2900         # 'query-balloon', but this might also fail for older versions...
2901         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
2902         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
2903     }
2904
2905     foreach my $vmid (keys %$list) {
2906         next if $opt_vmid && ($vmid ne $opt_vmid);
2907         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2908     }
2909
2910     return $res;
2911 }
2912
2913 sub conf_has_serial {
2914     my ($conf) = @_;
2915
2916     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2917         if ($conf->{"serial$i"}) {
2918             return 1;
2919         }
2920     }
2921
2922     return 0;
2923 }
2924
2925 sub conf_has_audio {
2926     my ($conf, $id) = @_;
2927
2928     $id //= 0;
2929     my $audio = $conf->{"audio$id"};
2930     return if !defined($audio);
2931
2932     my $audioproperties = parse_property_string($audio_fmt, $audio);
2933     my $audiodriver = $audioproperties->{driver} // 'spice';
2934
2935     return {
2936         dev => $audioproperties->{device},
2937         dev_id => "audiodev$id",
2938         backend => $audiodriver,
2939         backend_id => "$audiodriver-backend${id}",
2940     };
2941 }
2942
2943 sub audio_devs {
2944     my ($audio, $audiopciaddr, $machine_version) = @_;
2945
2946     my $devs = [];
2947
2948     my $id = $audio->{dev_id};
2949     my $audiodev = "";
2950     if (min_version($machine_version, 4, 2)) {
2951         $audiodev = ",audiodev=$audio->{backend_id}";
2952     }
2953
2954     if ($audio->{dev} eq 'AC97') {
2955         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
2956     } elsif ($audio->{dev} =~ /intel\-hda$/) {
2957         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
2958         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
2959         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
2960     } else {
2961         die "unkown audio device '$audio->{dev}', implement me!";
2962     }
2963
2964     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
2965
2966     return $devs;
2967 }
2968
2969 sub vga_conf_has_spice {
2970     my ($vga) = @_;
2971
2972     my $vgaconf = parse_vga($vga);
2973     my $vgatype = $vgaconf->{type};
2974     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
2975
2976     return $1 || 1;
2977 }
2978
2979 sub is_native($) {
2980     my ($arch) = @_;
2981     return get_host_arch() eq $arch;
2982 }
2983
2984 sub get_vm_arch {
2985     my ($conf) = @_;
2986     return $conf->{arch} // get_host_arch();
2987 }
2988
2989 my $default_machines = {
2990     x86_64 => 'pc',
2991     aarch64 => 'virt',
2992 };
2993
2994 sub get_installed_machine_version {
2995     my ($kvmversion) = @_;
2996     $kvmversion = kvm_user_version() if !defined($kvmversion);
2997     $kvmversion =~ m/^(\d+\.\d+)/;
2998     return $1;
2999 }
3000
3001 sub windows_get_pinned_machine_version {
3002     my ($machine, $base_version, $kvmversion) = @_;
3003
3004     my $pin_version = $base_version;
3005     if (!defined($base_version) ||
3006         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
3007     ) {
3008         $pin_version = get_installed_machine_version($kvmversion);
3009     }
3010     if (!$machine || $machine eq 'pc') {
3011         $machine = "pc-i440fx-$pin_version";
3012     } elsif ($machine eq 'q35') {
3013         $machine = "pc-q35-$pin_version";
3014     } elsif ($machine eq 'virt') {
3015         $machine = "virt-$pin_version";
3016     } else {
3017         warn "unknown machine type '$machine', not touching that!\n";
3018     }
3019
3020     return $machine;
3021 }
3022
3023 sub get_vm_machine {
3024     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3025
3026     my $machine = $forcemachine || $conf->{machine};
3027
3028     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3029         $kvmversion //= kvm_user_version();
3030         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3031         # layout which confuses windows quite a bit and may result in various regressions..
3032         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3033         if (windows_version($conf->{ostype})) {
3034             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3035         }
3036         $arch //= 'x86_64';
3037         $machine ||= $default_machines->{$arch};
3038         if ($add_pve_version) {
3039             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3040             $machine .= "+pve$pvever";
3041         }
3042     }
3043
3044     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3045         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3046         $machine = $1 if $is_pxe;
3047
3048         # for version-pinned machines that do not include a pve-version (e.g.
3049         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3050         $machine .= '+pve0';
3051
3052         $machine .= '.pxe' if $is_pxe;
3053     }
3054
3055     return $machine;
3056 }
3057
3058 sub get_ovmf_files($) {
3059     my ($arch) = @_;
3060
3061     my $ovmf = $OVMF->{$arch}
3062         or die "no OVMF images known for architecture '$arch'\n";
3063
3064     return @$ovmf;
3065 }
3066
3067 my $Arch2Qemu = {
3068     aarch64 => '/usr/bin/qemu-system-aarch64',
3069     x86_64 => '/usr/bin/qemu-system-x86_64',
3070 };
3071 sub get_command_for_arch($) {
3072     my ($arch) = @_;
3073     return '/usr/bin/kvm' if is_native($arch);
3074
3075     my $cmd = $Arch2Qemu->{$arch}
3076         or die "don't know how to emulate architecture '$arch'\n";
3077     return $cmd;
3078 }
3079
3080 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3081 # to use in a QEMU command line (-cpu element), first array_intersect the result
3082 # of query_supported_ with query_understood_. This is necessary because:
3083 #
3084 # a) query_understood_ returns flags the host cannot use and
3085 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3086 #    flags, but CPU settings - with most of them being flags. Those settings
3087 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3088 #
3089 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3090 # expensive. If you need the value returned from this, you can get it much
3091 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3092 # $accel being 'kvm' or 'tcg'.
3093 #
3094 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3095 # changes, automatically populating pmxcfs.
3096 #
3097 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3098 # since kvm and tcg machines support different flags
3099 #
3100 sub query_supported_cpu_flags {
3101     my ($arch) = @_;
3102
3103     $arch //= get_host_arch();
3104     my $default_machine = $default_machines->{$arch};
3105
3106     my $flags = {};
3107
3108     # FIXME: Once this is merged, the code below should work for ARM as well:
3109     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3110     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3111         $arch eq "aarch64";
3112
3113     my $kvm_supported = defined(kvm_version());
3114     my $qemu_cmd = get_command_for_arch($arch);
3115     my $fakevmid = -1;
3116     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3117
3118     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3119     my $query_supported_run_qemu = sub {
3120         my ($kvm) = @_;
3121
3122         my $flags = {};
3123         my $cmd = [
3124             $qemu_cmd,
3125             '-machine', $default_machine,
3126             '-display', 'none',
3127             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3128             '-mon', 'chardev=qmp,mode=control',
3129             '-pidfile', $pidfile,
3130             '-S', '-daemonize'
3131         ];
3132
3133         if (!$kvm) {
3134             push @$cmd, '-accel', 'tcg';
3135         }
3136
3137         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3138         die "QEMU flag querying VM exited with code " . $rc if $rc;
3139
3140         eval {
3141             my $cmd_result = mon_cmd(
3142                 $fakevmid,
3143                 'query-cpu-model-expansion',
3144                 type => 'full',
3145                 model => { name => 'host' }
3146             );
3147
3148             my $props = $cmd_result->{model}->{props};
3149             foreach my $prop (keys %$props) {
3150                 next if $props->{$prop} ne '1';
3151                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3152                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3153                 # We only keep those with underscores, to match /proc/cpuinfo
3154                 $prop =~ s/\.|-/_/g;
3155                 $flags->{$prop} = 1;
3156             }
3157         };
3158         my $err = $@;
3159
3160         # force stop with 10 sec timeout and 'nocheck'
3161         # always stop, even if QMP failed
3162         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3163
3164         die $err if $err;
3165
3166         return [ sort keys %$flags ];
3167     };
3168
3169     # We need to query QEMU twice, since KVM and TCG have different supported flags
3170     PVE::QemuConfig->lock_config($fakevmid, sub {
3171         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3172         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3173
3174         if ($kvm_supported) {
3175             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3176             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3177         }
3178     });
3179
3180     return $flags;
3181 }
3182
3183 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3184 my $understood_cpu_flag_dir = "/usr/share/kvm";
3185 sub query_understood_cpu_flags {
3186     my $arch = get_host_arch();
3187     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3188
3189     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3190         if ! -e $filepath;
3191
3192     my $raw = file_get_contents($filepath);
3193     $raw =~ s/^\s+|\s+$//g;
3194     my @flags = split(/\s+/, $raw);
3195
3196     return \@flags;
3197 }
3198
3199 my sub get_cpuunits {
3200     my ($conf) = @_;
3201     return $conf->{cpuunits} // (PVE::CGroup::cgroup_mode() == 2 ? 100 : 1024);
3202 }
3203 sub config_to_command {
3204     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3205         $pbs_backing) = @_;
3206
3207     my $cmd = [];
3208     my $globalFlags = [];
3209     my $machineFlags = [];
3210     my $rtcFlags = [];
3211     my $devices = [];
3212     my $pciaddr = '';
3213     my $bridges = {};
3214     my $ostype = $conf->{ostype};
3215     my $winversion = windows_version($ostype);
3216     my $kvm = $conf->{kvm};
3217     my $nodename = nodename();
3218
3219     my $arch = get_vm_arch($conf);
3220     my $kvm_binary = get_command_for_arch($arch);
3221     my $kvmver = kvm_user_version($kvm_binary);
3222
3223     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3224         $kvmver //= "undefined";
3225         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3226     }
3227
3228     my $add_pve_version = min_version($kvmver, 4, 1);
3229
3230     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3231     my $machine_version = extract_version($machine_type, $kvmver);
3232     $kvm //= 1 if is_native($arch);
3233
3234     $machine_version =~ m/(\d+)\.(\d+)/;
3235     my ($machine_major, $machine_minor) = ($1, $2);
3236
3237     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3238         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3239     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3240         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3241             ." please upgrade node '$nodename'\n"
3242     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3243         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3244         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3245             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3246             ." node '$nodename'\n";
3247     }
3248
3249     # if a specific +pve version is required for a feature, use $version_guard
3250     # instead of min_version to allow machines to be run with the minimum
3251     # required version
3252     my $required_pve_version = 0;
3253     my $version_guard = sub {
3254         my ($major, $minor, $pve) = @_;
3255         return 0 if !min_version($machine_version, $major, $minor, $pve);
3256         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3257         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3258         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3259         return 1;
3260     };
3261
3262     if ($kvm && !defined kvm_version()) {
3263         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3264             ." or enable in BIOS.\n";
3265     }
3266
3267     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3268     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3269     my $use_old_bios_files = undef;
3270     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3271
3272     my $cpuunits = get_cpuunits($conf);
3273
3274     push @$cmd, $kvm_binary;
3275
3276     push @$cmd, '-id', $vmid;
3277
3278     my $vmname = $conf->{name} || "vm$vmid";
3279
3280     push @$cmd, '-name', $vmname;
3281
3282     push @$cmd, '-no-shutdown';
3283
3284     my $use_virtio = 0;
3285
3286     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3287     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3288     push @$cmd, '-mon', "chardev=qmp,mode=control";
3289
3290     if (min_version($machine_version, 2, 12)) {
3291         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3292         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3293     }
3294
3295     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3296
3297     push @$cmd, '-daemonize';
3298
3299     if ($conf->{smbios1}) {
3300         my $smbios_conf = parse_smbios1($conf->{smbios1});
3301         if ($smbios_conf->{base64}) {
3302             # Do not pass base64 flag to qemu
3303             delete $smbios_conf->{base64};
3304             my $smbios_string = "";
3305             foreach my $key (keys %$smbios_conf) {
3306                 my $value;
3307                 if ($key eq "uuid") {
3308                     $value = $smbios_conf->{uuid}
3309                 } else {
3310                     $value = decode_base64($smbios_conf->{$key});
3311                 }
3312                 # qemu accepts any binary data, only commas need escaping by double comma
3313                 $value =~ s/,/,,/g;
3314                 $smbios_string .= "," . $key . "=" . $value if $value;
3315             }
3316             push @$cmd, '-smbios', "type=1" . $smbios_string;
3317         } else {
3318             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3319         }
3320     }
3321
3322     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3323         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3324         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3325
3326         my ($path, $format);
3327         my $read_only_str = '';
3328         if (my $efidisk = $conf->{efidisk0}) {
3329             my $d = parse_drive('efidisk0', $efidisk);
3330             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3331             $format = $d->{format};
3332             if ($storeid) {
3333                 $path = PVE::Storage::path($storecfg, $d->{file});
3334                 if (!defined($format)) {
3335                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3336                     $format = qemu_img_format($scfg, $volname);
3337                 }
3338             } else {
3339                 $path = $d->{file};
3340                 die "efidisk format must be specified\n"
3341                     if !defined($format);
3342             }
3343
3344             $read_only_str = ',readonly=on' if drive_is_read_only($conf, $d);
3345         } else {
3346             warn "no efidisk configured! Using temporary efivars disk.\n";
3347             $path = "/tmp/$vmid-ovmf.fd";
3348             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3349             $format = 'raw';
3350         }
3351
3352         my $size_str = "";
3353
3354         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3355             $size_str = ",size=" . (-s $ovmf_vars);
3356         }
3357
3358         # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3359         my $cache = "";
3360         if ($path =~ m/^rbd:/) {
3361                 $cache = ',cache=writeback';
3362                 $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3363         }
3364
3365         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code";
3366         push @$cmd, '-drive', "if=pflash,unit=1$cache,format=$format,id=drive-efidisk0$size_str,file=${path}${read_only_str}";
3367     }
3368
3369     # load q35 config
3370     if ($q35) {
3371         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3372         if (min_version($machine_version, 4, 0)) {
3373             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3374         } else {
3375             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3376         }
3377     }
3378
3379     if ($conf->{vmgenid}) {
3380         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3381     }
3382
3383     # add usb controllers
3384     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3385         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3386     push @$devices, @usbcontrollers if @usbcontrollers;
3387     my $vga = parse_vga($conf->{vga});
3388
3389     my $qxlnum = vga_conf_has_spice($conf->{vga});
3390     $vga->{type} = 'qxl' if $qxlnum;
3391
3392     if (!$vga->{type}) {
3393         if ($arch eq 'aarch64') {
3394             $vga->{type} = 'virtio';
3395         } elsif (min_version($machine_version, 2, 9)) {
3396             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3397         } else {
3398             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3399         }
3400     }
3401
3402     # enable absolute mouse coordinates (needed by vnc)
3403     my $tablet;
3404     if (defined($conf->{tablet})) {
3405         $tablet = $conf->{tablet};
3406     } else {
3407         $tablet = $defaults->{tablet};
3408         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3409         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3410     }
3411
3412     if ($tablet) {
3413         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3414         my $kbd = print_keyboarddevice_full($conf, $arch);
3415         push @$devices, '-device', $kbd if defined($kbd);
3416     }
3417
3418     my $bootorder = device_bootorder($conf);
3419
3420     # host pci device passthrough
3421     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3422         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3423
3424     # usb devices
3425     my $usb_dev_features = {};
3426     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3427
3428     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3429         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3430     push @$devices, @usbdevices if @usbdevices;
3431
3432     # serial devices
3433     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3434         if (my $path = $conf->{"serial$i"}) {
3435             if ($path eq 'socket') {
3436                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3437                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3438                 # On aarch64, serial0 is the UART device. Qemu only allows
3439                 # connecting UART devices via the '-serial' command line, as
3440                 # the device has a fixed slot on the hardware...
3441                 if ($arch eq 'aarch64' && $i == 0) {
3442                     push @$devices, '-serial', "chardev:serial$i";
3443                 } else {
3444                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3445                 }
3446             } else {
3447                 die "no such serial device\n" if ! -c $path;
3448                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3449                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3450             }
3451         }
3452     }
3453
3454     # parallel devices
3455     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3456         if (my $path = $conf->{"parallel$i"}) {
3457             die "no such parallel device\n" if ! -c $path;
3458             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3459             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3460             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3461         }
3462     }
3463
3464     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3465         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3466         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3467         push @$devices, @$audio_devs;
3468     }
3469
3470     my $sockets = 1;
3471     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3472     $sockets = $conf->{sockets} if  $conf->{sockets};
3473
3474     my $cores = $conf->{cores} || 1;
3475
3476     my $maxcpus = $sockets * $cores;
3477
3478     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3479
3480     my $allowed_vcpus = $cpuinfo->{cpus};
3481
3482     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3483         if ($allowed_vcpus < $maxcpus);
3484
3485     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3486
3487         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3488         for (my $i = 2; $i <= $vcpus; $i++)  {
3489             my $cpustr = print_cpu_device($conf,$i);
3490             push @$cmd, '-device', $cpustr;
3491         }
3492
3493     } else {
3494
3495         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3496     }
3497     push @$cmd, '-nodefaults';
3498
3499     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3500
3501     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3502
3503     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3504
3505     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3506         push @$devices, '-device', print_vga_device(
3507             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3508         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3509         push @$cmd,  '-vnc', "unix:$socket,password=on";
3510     } else {
3511         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3512         push @$cmd, '-nographic';
3513     }
3514
3515     # time drift fix
3516     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3517     my $useLocaltime = $conf->{localtime};
3518
3519     if ($winversion >= 5) { # windows
3520         $useLocaltime = 1 if !defined($conf->{localtime});
3521
3522         # use time drift fix when acpi is enabled
3523         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3524             $tdf = 1 if !defined($conf->{tdf});
3525         }
3526     }
3527
3528     if ($winversion >= 6) {
3529         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3530         push @$cmd, '-no-hpet';
3531     }
3532
3533     push @$rtcFlags, 'driftfix=slew' if $tdf;
3534
3535     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3536         push @$rtcFlags, "base=$conf->{startdate}";
3537     } elsif ($useLocaltime) {
3538         push @$rtcFlags, 'base=localtime';
3539     }
3540
3541     if ($forcecpu) {
3542         push @$cmd, '-cpu', $forcecpu;
3543     } else {
3544         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3545     }
3546
3547     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3548
3549     push @$cmd, '-S' if $conf->{freeze};
3550
3551     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3552
3553     my $guest_agent = parse_guest_agent($conf);
3554
3555     if ($guest_agent->{enabled}) {
3556         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3557         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3558
3559         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3560             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3561             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3562             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3563         } elsif ($guest_agent->{type} eq 'isa') {
3564             push @$devices, '-device', "isa-serial,chardev=qga0";
3565         }
3566     }
3567
3568     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3569     if ($rng && $version_guard->(4, 1, 2)) {
3570         check_rng_source($rng->{source});
3571
3572         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3573         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3574         my $limiter_str = "";
3575         if ($max_bytes) {
3576             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3577         }
3578
3579         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3580         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3581         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3582     }
3583
3584     my $spice_port;
3585
3586     if ($qxlnum) {
3587         if ($qxlnum > 1) {
3588             if ($winversion){
3589                 for (my $i = 1; $i < $qxlnum; $i++){
3590                     push @$devices, '-device', print_vga_device(
3591                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3592                 }
3593             } else {
3594                 # assume other OS works like Linux
3595                 my ($ram, $vram) = ("134217728", "67108864");
3596                 if ($vga->{memory}) {
3597                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3598                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3599                 }
3600                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3601                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3602             }
3603         }
3604
3605         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3606
3607         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3608         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3609         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3610
3611         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3612         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3613         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3614
3615         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3616         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3617
3618         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3619         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3620         if ($spice_enhancement->{foldersharing}) {
3621             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3622             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3623         }
3624
3625         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3626         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3627             if $spice_enhancement->{videostreaming};
3628
3629         push @$devices, '-spice', "$spice_opts";
3630     }
3631
3632     # enable balloon by default, unless explicitly disabled
3633     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3634         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3635         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3636     }
3637
3638     if ($conf->{watchdog}) {
3639         my $wdopts = parse_watchdog($conf->{watchdog});
3640         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3641         my $watchdog = $wdopts->{model} || 'i6300esb';
3642         push @$devices, '-device', "$watchdog$pciaddr";
3643         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3644     }
3645
3646     my $vollist = [];
3647     my $scsicontroller = {};
3648     my $ahcicontroller = {};
3649     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3650
3651     # Add iscsi initiator name if available
3652     if (my $initiator = get_initiator_name()) {
3653         push @$devices, '-iscsi', "initiator-name=$initiator";
3654     }
3655
3656     PVE::QemuConfig->foreach_volume($conf, sub {
3657         my ($ds, $drive) = @_;
3658
3659         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3660             check_volume_storage_type($storecfg, $drive->{file});
3661             push @$vollist, $drive->{file};
3662         }
3663
3664         # ignore efidisk here, already added in bios/fw handling code above
3665         return if $drive->{interface} eq 'efidisk';
3666
3667         $use_virtio = 1 if $ds =~ m/^virtio/;
3668
3669         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3670
3671         if ($drive->{interface} eq 'virtio'){
3672            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3673         }
3674
3675         if ($drive->{interface} eq 'scsi') {
3676
3677             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3678
3679             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3680                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3681
3682             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3683             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3684
3685             my $iothread = '';
3686             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3687                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3688                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3689             } elsif ($drive->{iothread}) {
3690                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3691             }
3692
3693             my $queues = '';
3694             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3695                 $queues = ",num_queues=$drive->{queues}";
3696             }
3697
3698             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3699                 if !$scsicontroller->{$controller};
3700             $scsicontroller->{$controller}=1;
3701         }
3702
3703         if ($drive->{interface} eq 'sata') {
3704             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3705             $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3706             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3707                 if !$ahcicontroller->{$controller};
3708             $ahcicontroller->{$controller}=1;
3709         }
3710
3711         my $pbs_conf = $pbs_backing->{$ds};
3712         my $pbs_name = undef;
3713         if ($pbs_conf) {
3714             $pbs_name = "drive-$ds-pbs";
3715             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
3716         }
3717
3718         my $drive_cmd = print_drive_commandline_full(
3719             $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
3720
3721         # extra protection for templates, but SATA and IDE don't support it..
3722         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
3723
3724         push @$devices, '-drive',$drive_cmd;
3725         push @$devices, '-device', print_drivedevice_full(
3726             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3727     });
3728
3729     for (my $i = 0; $i < $MAX_NETS; $i++) {
3730         my $netname = "net$i";
3731
3732         next if !$conf->{$netname};
3733         my $d = parse_net($conf->{$netname});
3734         next if !$d;
3735
3736         $use_virtio = 1 if $d->{model} eq 'virtio';
3737
3738         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
3739
3740         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
3741         push @$devices, '-netdev', $netdevfull;
3742
3743         my $netdevicefull = print_netdevice_full(
3744             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
3745
3746         push @$devices, '-device', $netdevicefull;
3747     }
3748
3749     if ($conf->{ivshmem}) {
3750         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3751
3752         my $bus;
3753         if ($q35) {
3754             $bus = print_pcie_addr("ivshmem");
3755         } else {
3756             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3757         }
3758
3759         my $ivshmem_name = $ivshmem->{name} // $vmid;
3760         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3761
3762         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3763         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
3764             .",size=$ivshmem->{size}M";
3765     }
3766
3767     # pci.4 is nested in pci.1
3768     $bridges->{1} = 1 if $bridges->{4};
3769
3770     if (!$q35) {
3771         # add pci bridges
3772         if (min_version($machine_version, 2, 3)) {
3773            $bridges->{1} = 1;
3774            $bridges->{2} = 1;
3775         }
3776
3777         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3778
3779     }
3780
3781     for my $k (sort {$b cmp $a} keys %$bridges) {
3782         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3783
3784         my $k_name = $k;
3785         if ($k == 2 && $legacy_igd) {
3786             $k_name = "$k-igd";
3787         }
3788         $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
3789
3790         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3791         if ($q35) {
3792             # add after -readconfig pve-q35.cfg
3793             splice @$devices, 2, 0, '-device', $devstr;
3794         } else {
3795             unshift @$devices, '-device', $devstr if $k > 0;
3796         }
3797     }
3798
3799     if (!$kvm) {
3800         push @$machineFlags, 'accel=tcg';
3801     }
3802
3803     my $machine_type_min = $machine_type;
3804     if ($add_pve_version) {
3805         $machine_type_min =~ s/\+pve\d+$//;
3806         $machine_type_min .= "+pve$required_pve_version";
3807     }
3808     push @$machineFlags, "type=${machine_type_min}";
3809
3810     push @$cmd, @$devices;
3811     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
3812     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
3813     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
3814
3815     if (my $vmstate = $conf->{vmstate}) {
3816         my $statepath = PVE::Storage::path($storecfg, $vmstate);
3817         push @$vollist, $vmstate;
3818         push @$cmd, '-loadstate', $statepath;
3819         print "activating and using '$vmstate' as vmstate\n";
3820     }
3821
3822     if (PVE::QemuConfig->is_template($conf)) {
3823         # needed to workaround base volumes being read-only
3824         push @$cmd, '-snapshot';
3825     }
3826
3827     # add custom args
3828     if ($conf->{args}) {
3829         my $aa = PVE::Tools::split_args($conf->{args});
3830         push @$cmd, @$aa;
3831     }
3832
3833     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3834 }
3835
3836 sub check_rng_source {
3837     my ($source) = @_;
3838
3839     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3840     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
3841         if ! -e $source;
3842
3843     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
3844     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
3845         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
3846         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
3847         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
3848             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
3849             ." to the host.\n";
3850     }
3851 }
3852
3853 sub spice_port {
3854     my ($vmid) = @_;
3855
3856     my $res = mon_cmd($vmid, 'query-spice');
3857
3858     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3859 }
3860
3861 sub vm_devices_list {
3862     my ($vmid) = @_;
3863
3864     my $res = mon_cmd($vmid, 'query-pci');
3865     my $devices_to_check = [];
3866     my $devices = {};
3867     foreach my $pcibus (@$res) {
3868         push @$devices_to_check, @{$pcibus->{devices}},
3869     }
3870
3871     while (@$devices_to_check) {
3872         my $to_check = [];
3873         for my $d (@$devices_to_check) {
3874             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3875             next if !$d->{'pci_bridge'};
3876
3877             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3878             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3879         }
3880         $devices_to_check = $to_check;
3881     }
3882
3883     my $resblock = mon_cmd($vmid, 'query-block');
3884     foreach my $block (@$resblock) {
3885         if($block->{device} =~ m/^drive-(\S+)/){
3886                 $devices->{$1} = 1;
3887         }
3888     }
3889
3890     my $resmice = mon_cmd($vmid, 'query-mice');
3891     foreach my $mice (@$resmice) {
3892         if ($mice->{name} eq 'QEMU HID Tablet') {
3893             $devices->{tablet} = 1;
3894             last;
3895         }
3896     }
3897
3898     # for usb devices there is no query-usb
3899     # but we can iterate over the entries in
3900     # qom-list path=/machine/peripheral
3901     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3902     foreach my $per (@$resperipheral) {
3903         if ($per->{name} =~ m/^usb\d+$/) {
3904             $devices->{$per->{name}} = 1;
3905         }
3906     }
3907
3908     return $devices;
3909 }
3910
3911 sub vm_deviceplug {
3912     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3913
3914     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3915
3916     my $devices_list = vm_devices_list($vmid);
3917     return 1 if defined($devices_list->{$deviceid});
3918
3919     # add PCI bridge if we need it for the device
3920     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
3921
3922     if ($deviceid eq 'tablet') {
3923
3924         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
3925
3926     } elsif ($deviceid eq 'keyboard') {
3927
3928         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
3929
3930     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3931
3932         die "usb hotplug currently not reliable\n";
3933         # since we can't reliably hot unplug all added usb devices and usb
3934         # passthrough breaks live migration we disable usb hotplugging for now
3935         #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3936
3937     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3938
3939         qemu_iothread_add($vmid, $deviceid, $device);
3940
3941         qemu_driveadd($storecfg, $vmid, $device);
3942         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3943
3944         qemu_deviceadd($vmid, $devicefull);
3945         eval { qemu_deviceaddverify($vmid, $deviceid); };
3946         if (my $err = $@) {
3947             eval { qemu_drivedel($vmid, $deviceid); };
3948             warn $@ if $@;
3949             die $err;
3950         }
3951
3952     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3953
3954
3955         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3956         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3957         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3958
3959         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3960
3961         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3962             qemu_iothread_add($vmid, $deviceid, $device);
3963             $devicefull .= ",iothread=iothread-$deviceid";
3964         }
3965
3966         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3967             $devicefull .= ",num_queues=$device->{queues}";
3968         }
3969
3970         qemu_deviceadd($vmid, $devicefull);
3971         qemu_deviceaddverify($vmid, $deviceid);
3972
3973     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3974
3975         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
3976         qemu_driveadd($storecfg, $vmid, $device);
3977
3978         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3979         eval { qemu_deviceadd($vmid, $devicefull); };
3980         if (my $err = $@) {
3981             eval { qemu_drivedel($vmid, $deviceid); };
3982             warn $@ if $@;
3983             die $err;
3984         }
3985
3986     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3987
3988         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
3989
3990         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
3991         my $use_old_bios_files = undef;
3992         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3993
3994         my $netdevicefull = print_netdevice_full(
3995             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
3996         qemu_deviceadd($vmid, $netdevicefull);
3997         eval {
3998             qemu_deviceaddverify($vmid, $deviceid);
3999             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4000         };
4001         if (my $err = $@) {
4002             eval { qemu_netdevdel($vmid, $deviceid); };
4003             warn $@ if $@;
4004             die $err;
4005         }
4006
4007     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4008
4009         my $bridgeid = $2;
4010         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4011         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4012
4013         qemu_deviceadd($vmid, $devicefull);
4014         qemu_deviceaddverify($vmid, $deviceid);
4015
4016     } else {
4017         die "can't hotplug device '$deviceid'\n";
4018     }
4019
4020     return 1;
4021 }
4022
4023 # fixme: this should raise exceptions on error!
4024 sub vm_deviceunplug {
4025     my ($vmid, $conf, $deviceid) = @_;
4026
4027     my $devices_list = vm_devices_list($vmid);
4028     return 1 if !defined($devices_list->{$deviceid});
4029
4030     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
4031     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4032
4033     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4034         qemu_devicedel($vmid, $deviceid);
4035     } elsif ($deviceid =~ m/^usb\d+$/) {
4036         die "usb hotplug currently not reliable\n";
4037         # when unplugging usb devices this way, there may be remaining usb
4038         # controllers/hubs so we disable it for now
4039         #qemu_devicedel($vmid, $deviceid);
4040         #qemu_devicedelverify($vmid, $deviceid);
4041     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4042         my $device = parse_drive($deviceid, $conf->{$deviceid});
4043
4044         qemu_devicedel($vmid, $deviceid);
4045         qemu_devicedelverify($vmid, $deviceid);
4046         qemu_drivedel($vmid, $deviceid);
4047         qemu_iothread_del($vmid, $deviceid, $device);
4048     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4049         qemu_devicedel($vmid, $deviceid);
4050         qemu_devicedelverify($vmid, $deviceid);
4051     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4052         my $device = parse_drive($deviceid, $conf->{$deviceid});
4053
4054         qemu_devicedel($vmid, $deviceid);
4055         qemu_drivedel($vmid, $deviceid);
4056         qemu_deletescsihw($conf, $vmid, $deviceid);
4057
4058         qemu_iothread_del($vmid, "virtioscsi$device->{index}", $device)
4059             if $conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single');
4060     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4061         qemu_devicedel($vmid, $deviceid);
4062         qemu_devicedelverify($vmid, $deviceid);
4063         qemu_netdevdel($vmid, $deviceid);
4064     } else {
4065         die "can't unplug device '$deviceid'\n";
4066     }
4067
4068     return 1;
4069 }
4070
4071 sub qemu_deviceadd {
4072     my ($vmid, $devicefull) = @_;
4073
4074     $devicefull = "driver=".$devicefull;
4075     my %options =  split(/[=,]/, $devicefull);
4076
4077     mon_cmd($vmid, "device_add" , %options);
4078 }
4079
4080 sub qemu_devicedel {
4081     my ($vmid, $deviceid) = @_;
4082
4083     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4084 }
4085
4086 sub qemu_iothread_add {
4087     my ($vmid, $deviceid, $device) = @_;
4088
4089     if ($device->{iothread}) {
4090         my $iothreads = vm_iothreads_list($vmid);
4091         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4092     }
4093 }
4094
4095 sub qemu_iothread_del {
4096     my ($vmid, $deviceid, $device) = @_;
4097
4098     if ($device->{iothread}) {
4099         my $iothreads = vm_iothreads_list($vmid);
4100         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4101     }
4102 }
4103
4104 sub qemu_objectadd {
4105     my ($vmid, $objectid, $qomtype) = @_;
4106
4107     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4108
4109     return 1;
4110 }
4111
4112 sub qemu_objectdel {
4113     my ($vmid, $objectid) = @_;
4114
4115     mon_cmd($vmid, "object-del", id => $objectid);
4116
4117     return 1;
4118 }
4119
4120 sub qemu_driveadd {
4121     my ($storecfg, $vmid, $device) = @_;
4122
4123     my $kvmver = get_running_qemu_version($vmid);
4124     my $io_uring = min_version($kvmver, 6, 0);
4125     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4126     $drive =~ s/\\/\\\\/g;
4127     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4128
4129     # If the command succeeds qemu prints: "OK"
4130     return 1 if $ret =~ m/OK/s;
4131
4132     die "adding drive failed: $ret\n";
4133 }
4134
4135 sub qemu_drivedel {
4136     my ($vmid, $deviceid) = @_;
4137
4138     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4139     $ret =~ s/^\s+//;
4140
4141     return 1 if $ret eq "";
4142
4143     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4144     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4145
4146     die "deleting drive $deviceid failed : $ret\n";
4147 }
4148
4149 sub qemu_deviceaddverify {
4150     my ($vmid, $deviceid) = @_;
4151
4152     for (my $i = 0; $i <= 5; $i++) {
4153          my $devices_list = vm_devices_list($vmid);
4154          return 1 if defined($devices_list->{$deviceid});
4155          sleep 1;
4156     }
4157
4158     die "error on hotplug device '$deviceid'\n";
4159 }
4160
4161
4162 sub qemu_devicedelverify {
4163     my ($vmid, $deviceid) = @_;
4164
4165     # need to verify that the device is correctly removed as device_del
4166     # is async and empty return is not reliable
4167
4168     for (my $i = 0; $i <= 5; $i++) {
4169          my $devices_list = vm_devices_list($vmid);
4170          return 1 if !defined($devices_list->{$deviceid});
4171          sleep 1;
4172     }
4173
4174     die "error on hot-unplugging device '$deviceid'\n";
4175 }
4176
4177 sub qemu_findorcreatescsihw {
4178     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4179
4180     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4181
4182     my $scsihwid="$controller_prefix$controller";
4183     my $devices_list = vm_devices_list($vmid);
4184
4185     if (!defined($devices_list->{$scsihwid})) {
4186         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4187     }
4188
4189     return 1;
4190 }
4191
4192 sub qemu_deletescsihw {
4193     my ($conf, $vmid, $opt) = @_;
4194
4195     my $device = parse_drive($opt, $conf->{$opt});
4196
4197     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4198         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4199         return 1;
4200     }
4201
4202     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4203
4204     my $devices_list = vm_devices_list($vmid);
4205     foreach my $opt (keys %{$devices_list}) {
4206         if (is_valid_drivename($opt)) {
4207             my $drive = parse_drive($opt, $conf->{$opt});
4208             if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4209                 return 1;
4210             }
4211         }
4212     }
4213
4214     my $scsihwid="scsihw$controller";
4215
4216     vm_deviceunplug($vmid, $conf, $scsihwid);
4217
4218     return 1;
4219 }
4220
4221 sub qemu_add_pci_bridge {
4222     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4223
4224     my $bridges = {};
4225
4226     my $bridgeid;
4227
4228     print_pci_addr($device, $bridges, $arch, $machine_type);
4229
4230     while (my ($k, $v) = each %$bridges) {
4231         $bridgeid = $k;
4232     }
4233     return 1 if !defined($bridgeid) || $bridgeid < 1;
4234
4235     my $bridge = "pci.$bridgeid";
4236     my $devices_list = vm_devices_list($vmid);
4237
4238     if (!defined($devices_list->{$bridge})) {
4239         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4240     }
4241
4242     return 1;
4243 }
4244
4245 sub qemu_set_link_status {
4246     my ($vmid, $device, $up) = @_;
4247
4248     mon_cmd($vmid, "set_link", name => $device,
4249                up => $up ? JSON::true : JSON::false);
4250 }
4251
4252 sub qemu_netdevadd {
4253     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4254
4255     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4256     my %options =  split(/[=,]/, $netdev);
4257
4258     if (defined(my $vhost = $options{vhost})) {
4259         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4260     }
4261
4262     if (defined(my $queues = $options{queues})) {
4263         $options{queues} = $queues + 0;
4264     }
4265
4266     mon_cmd($vmid, "netdev_add",  %options);
4267     return 1;
4268 }
4269
4270 sub qemu_netdevdel {
4271     my ($vmid, $deviceid) = @_;
4272
4273     mon_cmd($vmid, "netdev_del", id => $deviceid);
4274 }
4275
4276 sub qemu_usb_hotplug {
4277     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4278
4279     return if !$device;
4280
4281     # remove the old one first
4282     vm_deviceunplug($vmid, $conf, $deviceid);
4283
4284     # check if xhci controller is necessary and available
4285     if ($device->{usb3}) {
4286
4287         my $devicelist = vm_devices_list($vmid);
4288
4289         if (!$devicelist->{xhci}) {
4290             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4291             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4292         }
4293     }
4294     my $d = parse_usb_device($device->{host});
4295     $d->{usb3} = $device->{usb3};
4296
4297     # add the new one
4298     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4299 }
4300
4301 sub qemu_cpu_hotplug {
4302     my ($vmid, $conf, $vcpus) = @_;
4303
4304     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4305
4306     my $sockets = 1;
4307     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4308     $sockets = $conf->{sockets} if  $conf->{sockets};
4309     my $cores = $conf->{cores} || 1;
4310     my $maxcpus = $sockets * $cores;
4311
4312     $vcpus = $maxcpus if !$vcpus;
4313
4314     die "you can't add more vcpus than maxcpus\n"
4315         if $vcpus > $maxcpus;
4316
4317     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4318
4319     if ($vcpus < $currentvcpus) {
4320
4321         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4322
4323             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4324                 qemu_devicedel($vmid, "cpu$i");
4325                 my $retry = 0;
4326                 my $currentrunningvcpus = undef;
4327                 while (1) {
4328                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4329                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4330                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4331                     $retry++;
4332                     sleep 1;
4333                 }
4334                 #update conf after each succesfull cpu unplug
4335                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4336                 PVE::QemuConfig->write_config($vmid, $conf);
4337             }
4338         } else {
4339             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4340         }
4341
4342         return;
4343     }
4344
4345     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4346     die "vcpus in running vm does not match its configuration\n"
4347         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4348
4349     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4350
4351         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4352             my $cpustr = print_cpu_device($conf, $i);
4353             qemu_deviceadd($vmid, $cpustr);
4354
4355             my $retry = 0;
4356             my $currentrunningvcpus = undef;
4357             while (1) {
4358                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4359                 last if scalar(@{$currentrunningvcpus}) == $i;
4360                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4361                 sleep 1;
4362                 $retry++;
4363             }
4364             #update conf after each succesfull cpu hotplug
4365             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4366             PVE::QemuConfig->write_config($vmid, $conf);
4367         }
4368     } else {
4369
4370         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4371             mon_cmd($vmid, "cpu-add", id => int($i));
4372         }
4373     }
4374 }
4375
4376 sub qemu_block_set_io_throttle {
4377     my ($vmid, $deviceid,
4378         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4379         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4380         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4381         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4382
4383     return if !check_running($vmid) ;
4384
4385     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4386         bps => int($bps),
4387         bps_rd => int($bps_rd),
4388         bps_wr => int($bps_wr),
4389         iops => int($iops),
4390         iops_rd => int($iops_rd),
4391         iops_wr => int($iops_wr),
4392         bps_max => int($bps_max),
4393         bps_rd_max => int($bps_rd_max),
4394         bps_wr_max => int($bps_wr_max),
4395         iops_max => int($iops_max),
4396         iops_rd_max => int($iops_rd_max),
4397         iops_wr_max => int($iops_wr_max),
4398         bps_max_length => int($bps_max_length),
4399         bps_rd_max_length => int($bps_rd_max_length),
4400         bps_wr_max_length => int($bps_wr_max_length),
4401         iops_max_length => int($iops_max_length),
4402         iops_rd_max_length => int($iops_rd_max_length),
4403         iops_wr_max_length => int($iops_wr_max_length),
4404     );
4405
4406 }
4407
4408 sub qemu_block_resize {
4409     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4410
4411     my $running = check_running($vmid);
4412
4413     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4414
4415     return if !$running;
4416
4417     my $padding = (1024 - $size % 1024) % 1024;
4418     $size = $size + $padding;
4419
4420     mon_cmd(
4421         $vmid,
4422         "block_resize",
4423         device => $deviceid,
4424         size => int($size),
4425         timeout => 60,
4426     );
4427 }
4428
4429 sub qemu_volume_snapshot {
4430     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4431
4432     my $running = check_running($vmid);
4433
4434     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4435         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4436     } else {
4437         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4438     }
4439 }
4440
4441 sub qemu_volume_snapshot_delete {
4442     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4443
4444     my $running = check_running($vmid);
4445
4446     if($running) {
4447
4448         $running = undef;
4449         my $conf = PVE::QemuConfig->load_config($vmid);
4450         PVE::QemuConfig->foreach_volume($conf, sub {
4451             my ($ds, $drive) = @_;
4452             $running = 1 if $drive->{file} eq $volid;
4453         });
4454     }
4455
4456     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4457         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4458     } else {
4459         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4460     }
4461 }
4462
4463 sub set_migration_caps {
4464     my ($vmid, $savevm) = @_;
4465
4466     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4467
4468     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4469     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4470
4471     my $cap_ref = [];
4472
4473     my $enabled_cap = {
4474         "auto-converge" => 1,
4475         "xbzrle" => 1,
4476         "x-rdma-pin-all" => 0,
4477         "zero-blocks" => 0,
4478         "compress" => 0,
4479         "dirty-bitmaps" => $dirty_bitmaps,
4480     };
4481
4482     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4483
4484     for my $supported_capability (@$supported_capabilities) {
4485         push @$cap_ref, {
4486             capability => $supported_capability->{capability},
4487             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4488         };
4489     }
4490
4491     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4492 }
4493
4494 sub foreach_volid {
4495     my ($conf, $func, @param) = @_;
4496
4497     my $volhash = {};
4498
4499     my $test_volid = sub {
4500         my ($key, $drive, $snapname) = @_;
4501
4502         my $volid = $drive->{file};
4503         return if !$volid;
4504
4505         $volhash->{$volid}->{cdrom} //= 1;
4506         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4507
4508         my $replicate = $drive->{replicate} // 1;
4509         $volhash->{$volid}->{replicate} //= 0;
4510         $volhash->{$volid}->{replicate} = 1 if $replicate;
4511
4512         $volhash->{$volid}->{shared} //= 0;
4513         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4514
4515         $volhash->{$volid}->{referenced_in_config} //= 0;
4516         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4517
4518         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4519             if defined($snapname);
4520
4521         my $size = $drive->{size};
4522         $volhash->{$volid}->{size} //= $size if $size;
4523
4524         $volhash->{$volid}->{is_vmstate} //= 0;
4525         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4526
4527         $volhash->{$volid}->{is_unused} //= 0;
4528         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4529
4530         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4531     };
4532
4533     my $include_opts = {
4534         extra_keys => ['vmstate'],
4535         include_unused => 1,
4536     };
4537
4538     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4539     foreach my $snapname (keys %{$conf->{snapshots}}) {
4540         my $snap = $conf->{snapshots}->{$snapname};
4541         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4542     }
4543
4544     foreach my $volid (keys %$volhash) {
4545         &$func($volid, $volhash->{$volid}, @param);
4546     }
4547 }
4548
4549 my $fast_plug_option = {
4550     'lock' => 1,
4551     'name' => 1,
4552     'onboot' => 1,
4553     'shares' => 1,
4554     'startup' => 1,
4555     'description' => 1,
4556     'protection' => 1,
4557     'vmstatestorage' => 1,
4558     'hookscript' => 1,
4559     'tags' => 1,
4560 };
4561
4562 # hotplug changes in [PENDING]
4563 # $selection hash can be used to only apply specified options, for
4564 # example: { cores => 1 } (only apply changed 'cores')
4565 # $errors ref is used to return error messages
4566 sub vmconfig_hotplug_pending {
4567     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4568
4569     my $defaults = load_defaults();
4570     my $arch = get_vm_arch($conf);
4571     my $machine_type = get_vm_machine($conf, undef, $arch);
4572
4573     # commit values which do not have any impact on running VM first
4574     # Note: those option cannot raise errors, we we do not care about
4575     # $selection and always apply them.
4576
4577     my $add_error = sub {
4578         my ($opt, $msg) = @_;
4579         $errors->{$opt} = "hotplug problem - $msg";
4580     };
4581
4582     my $changes = 0;
4583     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4584         if ($fast_plug_option->{$opt}) {
4585             $conf->{$opt} = $conf->{pending}->{$opt};
4586             delete $conf->{pending}->{$opt};
4587             $changes = 1;
4588         }
4589     }
4590
4591     if ($changes) {
4592         PVE::QemuConfig->write_config($vmid, $conf);
4593     }
4594
4595     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4596
4597     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4598     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4599     foreach my $opt (sort keys %$pending_delete_hash) {
4600         next if $selection && !$selection->{$opt};
4601         my $force = $pending_delete_hash->{$opt}->{force};
4602         eval {
4603             if ($opt eq 'hotplug') {
4604                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4605             } elsif ($opt eq 'tablet') {
4606                 die "skip\n" if !$hotplug_features->{usb};
4607                 if ($defaults->{tablet}) {
4608                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4609                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4610                         if $arch eq 'aarch64';
4611                 } else {
4612                     vm_deviceunplug($vmid, $conf, 'tablet');
4613                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4614                 }
4615             } elsif ($opt =~ m/^usb\d+/) {
4616                 die "skip\n";
4617                 # since we cannot reliably hot unplug usb devices we are disabling it
4618                 #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4619                 #vm_deviceunplug($vmid, $conf, $opt);
4620             } elsif ($opt eq 'vcpus') {
4621                 die "skip\n" if !$hotplug_features->{cpu};
4622                 qemu_cpu_hotplug($vmid, $conf, undef);
4623             } elsif ($opt eq 'balloon') {
4624                 # enable balloon device is not hotpluggable
4625                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4626                 # here we reset the ballooning value to memory
4627                 my $balloon = $conf->{memory} || $defaults->{memory};
4628                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4629             } elsif ($fast_plug_option->{$opt}) {
4630                 # do nothing
4631             } elsif ($opt =~ m/^net(\d+)$/) {
4632                 die "skip\n" if !$hotplug_features->{network};
4633                 vm_deviceunplug($vmid, $conf, $opt);
4634             } elsif (is_valid_drivename($opt)) {
4635                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4636                 vm_deviceunplug($vmid, $conf, $opt);
4637                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4638             } elsif ($opt =~ m/^memory$/) {
4639                 die "skip\n" if !$hotplug_features->{memory};
4640                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4641             } elsif ($opt eq 'cpuunits') {
4642                 $cgroup->change_cpu_shares(undef, 1024);
4643             } elsif ($opt eq 'cpulimit') {
4644                 $cgroup->change_cpu_quota(-1, 100000);
4645             } else {
4646                 die "skip\n";
4647             }
4648         };
4649         if (my $err = $@) {
4650             &$add_error($opt, $err) if $err ne "skip\n";
4651         } else {
4652             delete $conf->{$opt};
4653             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4654         }
4655     }
4656
4657     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4658     $apply_pending_cloudinit = sub {
4659         return if $apply_pending_cloudinit_done; # once is enough
4660         $apply_pending_cloudinit_done = 1; # once is enough
4661
4662         my ($key, $value) = @_;
4663
4664         my @cloudinit_opts = keys %$confdesc_cloudinit;
4665         foreach my $opt (keys %{$conf->{pending}}) {
4666             next if !grep { $_ eq $opt } @cloudinit_opts;
4667             $conf->{$opt} = delete $conf->{pending}->{$opt};
4668         }
4669
4670         my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4671         foreach my $opt (sort keys %$pending_delete_hash) {
4672             next if !grep { $_ eq $opt } @cloudinit_opts;
4673             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4674             delete $conf->{$opt};
4675         }
4676
4677         my $new_conf = { %$conf };
4678         $new_conf->{$key} = $value;
4679         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4680     };
4681
4682     foreach my $opt (keys %{$conf->{pending}}) {
4683         next if $selection && !$selection->{$opt};
4684         my $value = $conf->{pending}->{$opt};
4685         eval {
4686             if ($opt eq 'hotplug') {
4687                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4688             } elsif ($opt eq 'tablet') {
4689                 die "skip\n" if !$hotplug_features->{usb};
4690                 if ($value == 1) {
4691                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4692                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4693                         if $arch eq 'aarch64';
4694                 } elsif ($value == 0) {
4695                     vm_deviceunplug($vmid, $conf, 'tablet');
4696                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4697                 }
4698             } elsif ($opt =~ m/^usb\d+$/) {
4699                 die "skip\n";
4700                 # since we cannot reliably hot unplug usb devices we disable it for now
4701                 #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4702                 #my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4703                 #die "skip\n" if !$d;
4704                 #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4705             } elsif ($opt eq 'vcpus') {
4706                 die "skip\n" if !$hotplug_features->{cpu};
4707                 qemu_cpu_hotplug($vmid, $conf, $value);
4708             } elsif ($opt eq 'balloon') {
4709                 # enable/disable balloning device is not hotpluggable
4710                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4711                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4712                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4713
4714                 # allow manual ballooning if shares is set to zero
4715                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4716                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4717                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4718                 }
4719             } elsif ($opt =~ m/^net(\d+)$/) {
4720                 # some changes can be done without hotplug
4721                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4722                                     $vmid, $opt, $value, $arch, $machine_type);
4723             } elsif (is_valid_drivename($opt)) {
4724                 die "skip\n" if $opt eq 'efidisk0';
4725                 # some changes can be done without hotplug
4726                 my $drive = parse_drive($opt, $value);
4727                 if (drive_is_cloudinit($drive)) {
4728                     &$apply_pending_cloudinit($opt, $value);
4729                 }
4730                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4731                                      $vmid, $opt, $value, $arch, $machine_type);
4732             } elsif ($opt =~ m/^memory$/) { #dimms
4733                 die "skip\n" if !$hotplug_features->{memory};
4734                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4735             } elsif ($opt eq 'cpuunits') {
4736                 $cgroup->change_cpu_shares($conf->{pending}->{$opt}, 1024);
4737             } elsif ($opt eq 'cpulimit') {
4738                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4739                 $cgroup->change_cpu_quota($cpulimit, 100000);
4740             } else {
4741                 die "skip\n";  # skip non-hot-pluggable options
4742             }
4743         };
4744         if (my $err = $@) {
4745             &$add_error($opt, $err) if $err ne "skip\n";
4746         } else {
4747             $conf->{$opt} = $value;
4748             delete $conf->{pending}->{$opt};
4749         }
4750     }
4751
4752     PVE::QemuConfig->write_config($vmid, $conf);
4753 }
4754
4755 sub try_deallocate_drive {
4756     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4757
4758     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4759         my $volid = $drive->{file};
4760         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4761             my $sid = PVE::Storage::parse_volume_id($volid);
4762             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4763
4764             # check if the disk is really unused
4765             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4766                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4767             PVE::Storage::vdisk_free($storecfg, $volid);
4768             return 1;
4769         } else {
4770             # If vm is not owner of this disk remove from config
4771             return 1;
4772         }
4773     }
4774
4775     return;
4776 }
4777
4778 sub vmconfig_delete_or_detach_drive {
4779     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4780
4781     my $drive = parse_drive($opt, $conf->{$opt});
4782
4783     my $rpcenv = PVE::RPCEnvironment::get();
4784     my $authuser = $rpcenv->get_user();
4785
4786     if ($force) {
4787         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4788         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4789     } else {
4790         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4791     }
4792 }
4793
4794
4795
4796 sub vmconfig_apply_pending {
4797     my ($vmid, $conf, $storecfg, $errors) = @_;
4798
4799     my $add_apply_error = sub {
4800         my ($opt, $msg) = @_;
4801         my $err_msg = "unable to apply pending change $opt : $msg";
4802         $errors->{$opt} = $err_msg;
4803         warn $err_msg;
4804     };
4805
4806     # cold plug
4807
4808     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4809     foreach my $opt (sort keys %$pending_delete_hash) {
4810         my $force = $pending_delete_hash->{$opt}->{force};
4811         eval {
4812             if ($opt =~ m/^unused/) {
4813                 die "internal error";
4814             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4815                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4816             }
4817         };
4818         if (my $err = $@) {
4819             $add_apply_error->($opt, $err);
4820         } else {
4821             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4822             delete $conf->{$opt};
4823         }
4824     }
4825
4826     PVE::QemuConfig->cleanup_pending($conf);
4827
4828     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4829         next if $opt eq 'delete'; # just to be sure
4830         eval {
4831             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4832                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4833             }
4834         };
4835         if (my $err = $@) {
4836             $add_apply_error->($opt, $err);
4837         } else {
4838             $conf->{$opt} = delete $conf->{pending}->{$opt};
4839         }
4840     }
4841
4842     # write all changes at once to avoid unnecessary i/o
4843     PVE::QemuConfig->write_config($vmid, $conf);
4844 }
4845
4846 sub vmconfig_update_net {
4847     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4848
4849     my $newnet = parse_net($value);
4850
4851     if ($conf->{$opt}) {
4852         my $oldnet = parse_net($conf->{$opt});
4853
4854         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4855             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4856             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4857             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4858
4859             # for non online change, we try to hot-unplug
4860             die "skip\n" if !$hotplug;
4861             vm_deviceunplug($vmid, $conf, $opt);
4862         } else {
4863
4864             die "internal error" if $opt !~ m/net(\d+)/;
4865             my $iface = "tap${vmid}i$1";
4866
4867             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4868                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4869                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4870                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4871                 PVE::Network::tap_unplug($iface);
4872
4873                 if ($have_sdn) {
4874                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4875                 } else {
4876                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4877                 }
4878             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4879                 # Rate can be applied on its own but any change above needs to
4880                 # include the rate in tap_plug since OVS resets everything.
4881                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4882             }
4883
4884             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4885                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4886             }
4887
4888             return 1;
4889         }
4890     }
4891
4892     if ($hotplug) {
4893         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4894     } else {
4895         die "skip\n";
4896     }
4897 }
4898
4899 sub vmconfig_update_disk {
4900     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4901
4902     my $drive = parse_drive($opt, $value);
4903
4904     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
4905         my $media = $drive->{media} || 'disk';
4906         my $oldmedia = $old_drive->{media} || 'disk';
4907         die "unable to change media type\n" if $media ne $oldmedia;
4908
4909         if (!drive_is_cdrom($old_drive)) {
4910
4911             if ($drive->{file} ne $old_drive->{file}) {
4912
4913                 die "skip\n" if !$hotplug;
4914
4915                 # unplug and register as unused
4916                 vm_deviceunplug($vmid, $conf, $opt);
4917                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4918
4919             } else {
4920                 # update existing disk
4921
4922                 # skip non hotpluggable value
4923                 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4924                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4925                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4926                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
4927                     safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
4928                     die "skip\n";
4929                 }
4930
4931                 # apply throttle
4932                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4933                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4934                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4935                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4936                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4937                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4938                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4939                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4940                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4941                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4942                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4943                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4944                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4945                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4946                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4947                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4948                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4949                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4950
4951                     qemu_block_set_io_throttle(
4952                         $vmid,"drive-$opt",
4953                         ($drive->{mbps} || 0)*1024*1024,
4954                         ($drive->{mbps_rd} || 0)*1024*1024,
4955                         ($drive->{mbps_wr} || 0)*1024*1024,
4956                          $drive->{iops} || 0,
4957                          $drive->{iops_rd} || 0,
4958                          $drive->{iops_wr} || 0,
4959                         ($drive->{mbps_max} || 0)*1024*1024,
4960                         ($drive->{mbps_rd_max} || 0)*1024*1024,
4961                         ($drive->{mbps_wr_max} || 0)*1024*1024,
4962                          $drive->{iops_max} || 0,
4963                          $drive->{iops_rd_max} || 0,
4964                          $drive->{iops_wr_max} || 0,
4965                          $drive->{bps_max_length} || 1,
4966                          $drive->{bps_rd_max_length} || 1,
4967                          $drive->{bps_wr_max_length} || 1,
4968                          $drive->{iops_max_length} || 1,
4969                          $drive->{iops_rd_max_length} || 1,
4970                          $drive->{iops_wr_max_length} || 1,
4971                     );
4972
4973                 }
4974
4975                 return 1;
4976             }
4977
4978         } else { # cdrom
4979
4980             if ($drive->{file} eq 'none') {
4981                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4982                 if (drive_is_cloudinit($old_drive)) {
4983                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4984                 }
4985             } else {
4986                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4987
4988                 # force eject if locked
4989                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4990
4991                 if ($path) {
4992                     mon_cmd($vmid, "blockdev-change-medium",
4993                         id => "$opt", filename => "$path");
4994                 }
4995             }
4996
4997             return 1;
4998         }
4999     }
5000
5001     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5002     # hotplug new disks
5003     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5004     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5005 }
5006
5007 # called in locked context by incoming migration
5008 sub vm_migrate_get_nbd_disks {
5009     my ($storecfg, $conf, $replicated_volumes) = @_;
5010
5011     my $local_volumes = {};
5012     PVE::QemuConfig->foreach_volume($conf, sub {
5013         my ($ds, $drive) = @_;
5014
5015         return if drive_is_cdrom($drive);
5016
5017         my $volid = $drive->{file};
5018
5019         return if !$volid;
5020
5021         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5022
5023         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5024         return if $scfg->{shared};
5025
5026         # replicated disks re-use existing state via bitmap
5027         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5028         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
5029     });
5030     return $local_volumes;
5031 }
5032
5033 # called in locked context by incoming migration
5034 sub vm_migrate_alloc_nbd_disks {
5035     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5036
5037     my $format = undef;
5038
5039     my $nbd = {};
5040     foreach my $opt (sort keys %$source_volumes) {
5041         my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
5042
5043         if ($use_existing) {
5044             $nbd->{$opt}->{drivestr} = print_drive($drive);
5045             $nbd->{$opt}->{volid} = $volid;
5046             $nbd->{$opt}->{replicated} = 1;
5047             next;
5048         }
5049
5050         # If a remote storage is specified and the format of the original
5051         # volume is not available there, fall back to the default format.
5052         # Otherwise use the same format as the original.
5053         if (!$storagemap->{identity}) {
5054             $storeid = map_storage($storagemap, $storeid);
5055             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5056             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5057             my $fileFormat = qemu_img_format($scfg, $volname);
5058             $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
5059         } else {
5060             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5061             $format = qemu_img_format($scfg, $volname);
5062         }
5063
5064         my $size = $drive->{size} / 1024;
5065         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5066         my $newdrive = $drive;
5067         $newdrive->{format} = $format;
5068         $newdrive->{file} = $newvolid;
5069         my $drivestr = print_drive($newdrive);
5070         $nbd->{$opt}->{drivestr} = $drivestr;
5071         $nbd->{$opt}->{volid} = $newvolid;
5072     }
5073
5074     return $nbd;
5075 }
5076
5077 # see vm_start_nolock for parameters, additionally:
5078 # migrate_opts:
5079 #   storagemap = parsed storage map for allocating NBD disks
5080 sub vm_start {
5081     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5082
5083     return PVE::QemuConfig->lock_config($vmid, sub {
5084         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5085
5086         die "you can't start a vm if it's a template\n"
5087             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5088
5089         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5090         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5091
5092         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5093
5094         if ($has_backup_lock && $running) {
5095             # a backup is currently running, attempt to start the guest in the
5096             # existing QEMU instance
5097             return vm_resume($vmid);
5098         }
5099
5100         PVE::QemuConfig->check_lock($conf)
5101             if !($params->{skiplock} || $has_suspended_lock);
5102
5103         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5104
5105         die "VM $vmid already running\n" if $running;
5106
5107         if (my $storagemap = $migrate_opts->{storagemap}) {
5108             my $replicated = $migrate_opts->{replicated_volumes};
5109             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5110             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5111
5112             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5113                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5114             }
5115         }
5116
5117         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5118     });
5119 }
5120
5121
5122 # params:
5123 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5124 #   skiplock => 0/1, skip checking for config lock
5125 #   skiptemplate => 0/1, skip checking whether VM is template
5126 #   forcemachine => to force Qemu machine (rollback/migration)
5127 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5128 #   timeout => in seconds
5129 #   paused => start VM in paused state (backup)
5130 #   resume => resume from hibernation
5131 #   pbs-backing => {
5132 #      sata0 => {
5133 #         repository
5134 #         snapshot
5135 #         keyfile
5136 #         archive
5137 #      },
5138 #      virtio2 => ...
5139 #   }
5140 # migrate_opts:
5141 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5142 #   migratedfrom => source node
5143 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5144 #   network => CIDR of migration network
5145 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5146 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5147 #   replicated_volumes = which volids should be re-used with bitmaps for nbd migration
5148 sub vm_start_nolock {
5149     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5150
5151     my $statefile = $params->{statefile};
5152     my $resume = $params->{resume};
5153
5154     my $migratedfrom = $migrate_opts->{migratedfrom};
5155     my $migration_type = $migrate_opts->{type};
5156
5157     my $res = {};
5158
5159     # clean up leftover reboot request files
5160     eval { clear_reboot_request($vmid); };
5161     warn $@ if $@;
5162
5163     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5164         vmconfig_apply_pending($vmid, $conf, $storecfg);
5165         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5166     }
5167
5168     # don't regenerate the ISO if the VM is started as part of a live migration
5169     # this way we can reuse the old ISO with the correct config
5170     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid) if !$migratedfrom;
5171
5172     my $defaults = load_defaults();
5173
5174     # set environment variable useful inside network script
5175     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5176
5177     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5178
5179     my $forcemachine = $params->{forcemachine};
5180     my $forcecpu = $params->{forcecpu};
5181     if ($resume) {
5182         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5183         $forcemachine = $conf->{runningmachine};
5184         $forcecpu = $conf->{runningcpu};
5185         print "Resuming suspended VM\n";
5186     }
5187
5188     my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid,
5189         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5190
5191     my $migration_ip;
5192     my $get_migration_ip = sub {
5193         my ($nodename) = @_;
5194
5195         return $migration_ip if defined($migration_ip);
5196
5197         my $cidr = $migrate_opts->{network};
5198
5199         if (!defined($cidr)) {
5200             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5201             $cidr = $dc_conf->{migration}->{network};
5202         }
5203
5204         if (defined($cidr)) {
5205             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5206
5207             die "could not get IP: no address configured on local " .
5208                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5209
5210             die "could not get IP: multiple addresses configured on local " .
5211                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5212
5213             $migration_ip = @$ips[0];
5214         }
5215
5216         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5217             if !defined($migration_ip);
5218
5219         return $migration_ip;
5220     };
5221
5222     my $migrate_uri;
5223     if ($statefile) {
5224         if ($statefile eq 'tcp') {
5225             my $localip = "localhost";
5226             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5227             my $nodename = nodename();
5228
5229             if (!defined($migration_type)) {
5230                 if (defined($datacenterconf->{migration}->{type})) {
5231                     $migration_type = $datacenterconf->{migration}->{type};
5232                 } else {
5233                     $migration_type = 'secure';
5234                 }
5235             }
5236
5237             if ($migration_type eq 'insecure') {
5238                 $localip = $get_migration_ip->($nodename);
5239                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5240             }
5241
5242             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5243             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5244             $migrate_uri = "tcp:${localip}:${migrate_port}";
5245             push @$cmd, '-incoming', $migrate_uri;
5246             push @$cmd, '-S';
5247
5248         } elsif ($statefile eq 'unix') {
5249             # should be default for secure migrations as a ssh TCP forward
5250             # tunnel is not deterministic reliable ready and fails regurarly
5251             # to set up in time, so use UNIX socket forwards
5252             my $socket_addr = "/run/qemu-server/$vmid.migrate";
5253             unlink $socket_addr;
5254
5255             $migrate_uri = "unix:$socket_addr";
5256
5257             push @$cmd, '-incoming', $migrate_uri;
5258             push @$cmd, '-S';
5259
5260         } elsif (-e $statefile) {
5261             push @$cmd, '-loadstate', $statefile;
5262         } else {
5263             my $statepath = PVE::Storage::path($storecfg, $statefile);
5264             push @$vollist, $statefile;
5265             push @$cmd, '-loadstate', $statepath;
5266         }
5267     } elsif ($params->{paused}) {
5268         push @$cmd, '-S';
5269     }
5270
5271     # host pci devices
5272     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5273       my $d = parse_hostpci($conf->{"hostpci$i"});
5274       next if !$d;
5275       my $pcidevices = $d->{pciid};
5276       foreach my $pcidevice (@$pcidevices) {
5277             my $pciid = $pcidevice->{id};
5278
5279             my $info = PVE::SysFSTools::pci_device_info("$pciid");
5280             die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5281             die "no pci device info for device '$pciid'\n" if !$info;
5282
5283             if ($d->{mdev}) {
5284                 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5285                 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5286             } else {
5287                 die "can't unbind/bind PCI group to VFIO '$pciid'\n"
5288                     if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5289                 die "can't reset PCI device '$pciid'\n"
5290                     if $info->{has_fl_reset} && !PVE::SysFSTools::pci_dev_reset($info);
5291             }
5292       }
5293     }
5294
5295     PVE::Storage::activate_volumes($storecfg, $vollist);
5296
5297     eval {
5298         run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5299             outfunc => sub {}, errfunc => sub {});
5300     };
5301     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5302     # timeout should be more than enough here...
5303     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5304
5305     my $cpuunits = get_cpuunits($conf);
5306
5307     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5308     my %run_params = (
5309         timeout => $statefile ? undef : $start_timeout,
5310         umask => 0077,
5311         noerr => 1,
5312     );
5313
5314     # when migrating, prefix QEMU output so other side can pick up any
5315     # errors that might occur and show the user
5316     if ($migratedfrom) {
5317         $run_params{quiet} = 1;
5318         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5319     }
5320
5321     my %properties = (
5322         Slice => 'qemu.slice',
5323         KillMode => 'process',
5324         SendSIGKILL => 0,
5325         TimeoutStopUSec => ULONG_MAX, # infinity
5326     );
5327
5328     if (PVE::CGroup::cgroup_mode() == 2) {
5329         $cpuunits = 10000 if $cpuunits >= 10000; # else we get an error
5330         $properties{CPUWeight} = $cpuunits;
5331     } else {
5332         $properties{CPUShares} = $cpuunits;
5333     }
5334
5335     if (my $cpulimit = $conf->{cpulimit}) {
5336         $properties{CPUQuota} = int($cpulimit * 100);
5337     }
5338     $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5339
5340     my $run_qemu = sub {
5341         PVE::Tools::run_fork sub {
5342             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5343
5344             my $exitcode = run_command($cmd, %run_params);
5345             die "QEMU exited with code $exitcode\n" if $exitcode;
5346         };
5347     };
5348
5349     if ($conf->{hugepages}) {
5350
5351         my $code = sub {
5352             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5353             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5354
5355             PVE::QemuServer::Memory::hugepages_mount();
5356             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5357
5358             eval { $run_qemu->() };
5359             if (my $err = $@) {
5360                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5361                     if !$conf->{keephugepages};
5362                 die $err;
5363             }
5364
5365             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5366                 if !$conf->{keephugepages};
5367         };
5368         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5369
5370     } else {
5371         eval { $run_qemu->() };
5372     }
5373
5374     if (my $err = $@) {
5375         # deactivate volumes if start fails
5376         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5377         die "start failed: $err";
5378     }
5379
5380     print "migration listens on $migrate_uri\n" if $migrate_uri;
5381     $res->{migrate_uri} = $migrate_uri;
5382
5383     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5384         eval { mon_cmd($vmid, "cont"); };
5385         warn $@ if $@;
5386     }
5387
5388     #start nbd server for storage migration
5389     if (my $nbd = $migrate_opts->{nbd}) {
5390         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5391
5392         my $migrate_storage_uri;
5393         # nbd_protocol_version > 0 for unix socket support
5394         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5395             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5396             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5397             $migrate_storage_uri = "nbd:unix:$socket_path";
5398         } else {
5399             my $nodename = nodename();
5400             my $localip = $get_migration_ip->($nodename);
5401             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5402             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5403
5404             mon_cmd($vmid, "nbd-server-start", addr => {
5405                 type => 'inet',
5406                 data => {
5407                     host => "${localip}",
5408                     port => "${storage_migrate_port}",
5409                 },
5410             });
5411             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5412             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5413         }
5414
5415         $res->{migrate_storage_uri} = $migrate_storage_uri;
5416
5417         foreach my $opt (sort keys %$nbd) {
5418             my $drivestr = $nbd->{$opt}->{drivestr};
5419             my $volid = $nbd->{$opt}->{volid};
5420             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5421             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5422             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5423             print "re-using replicated volume: $opt - $volid\n"
5424                 if $nbd->{$opt}->{replicated};
5425
5426             $res->{drives}->{$opt} = $nbd->{$opt};
5427             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5428         }
5429     }
5430
5431     if ($migratedfrom) {
5432         eval {
5433             set_migration_caps($vmid);
5434         };
5435         warn $@ if $@;
5436
5437         if ($spice_port) {
5438             print "spice listens on port $spice_port\n";
5439             $res->{spice_port} = $spice_port;
5440             if ($migrate_opts->{spice_ticket}) {
5441                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5442                     $migrate_opts->{spice_ticket});
5443                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5444             }
5445         }
5446
5447     } else {
5448         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5449             if !$statefile && $conf->{balloon};
5450
5451         foreach my $opt (keys %$conf) {
5452             next if $opt !~  m/^net\d+$/;
5453             my $nicconf = parse_net($conf->{$opt});
5454             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5455         }
5456     }
5457
5458     mon_cmd($vmid, 'qom-set',
5459                 path => "machine/peripheral/balloon0",
5460                 property => "guest-stats-polling-interval",
5461                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5462
5463     if ($resume) {
5464         print "Resumed VM, removing state\n";
5465         if (my $vmstate = $conf->{vmstate}) {
5466             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5467             PVE::Storage::vdisk_free($storecfg, $vmstate);
5468         }
5469         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5470         PVE::QemuConfig->write_config($vmid, $conf);
5471     }
5472
5473     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5474
5475     return $res;
5476 }
5477
5478 sub vm_commandline {
5479     my ($storecfg, $vmid, $snapname) = @_;
5480
5481     my $conf = PVE::QemuConfig->load_config($vmid);
5482     my $forcemachine;
5483     my $forcecpu;
5484
5485     if ($snapname) {
5486         my $snapshot = $conf->{snapshots}->{$snapname};
5487         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5488
5489         # check for machine or CPU overrides in snapshot
5490         $forcemachine = $snapshot->{runningmachine};
5491         $forcecpu = $snapshot->{runningcpu};
5492
5493         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5494
5495         $conf = $snapshot;
5496     }
5497
5498     my $defaults = load_defaults();
5499
5500     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults,
5501         $forcemachine, $forcecpu);
5502
5503     return PVE::Tools::cmd2string($cmd);
5504 }
5505
5506 sub vm_reset {
5507     my ($vmid, $skiplock) = @_;
5508
5509     PVE::QemuConfig->lock_config($vmid, sub {
5510
5511         my $conf = PVE::QemuConfig->load_config($vmid);
5512
5513         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5514
5515         mon_cmd($vmid, "system_reset");
5516     });
5517 }
5518
5519 sub get_vm_volumes {
5520     my ($conf) = @_;
5521
5522     my $vollist = [];
5523     foreach_volid($conf, sub {
5524         my ($volid, $attr) = @_;
5525
5526         return if $volid =~ m|^/|;
5527
5528         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5529         return if !$sid;
5530
5531         push @$vollist, $volid;
5532     });
5533
5534     return $vollist;
5535 }
5536
5537 sub vm_stop_cleanup {
5538     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5539
5540     eval {
5541
5542         if (!$keepActive) {
5543             my $vollist = get_vm_volumes($conf);
5544             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5545         }
5546
5547         foreach my $ext (qw(mon qmp pid vnc qga)) {
5548             unlink "/var/run/qemu-server/${vmid}.$ext";
5549         }
5550
5551         if ($conf->{ivshmem}) {
5552             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5553             # just delete it for now, VMs which have this already open do not
5554             # are affected, but new VMs will get a separated one. If this
5555             # becomes an issue we either add some sort of ref-counting or just
5556             # add a "don't delete on stop" flag to the ivshmem format.
5557             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5558         }
5559
5560         foreach my $key (keys %$conf) {
5561             next if $key !~ m/^hostpci(\d+)$/;
5562             my $hostpciindex = $1;
5563             my $d = parse_hostpci($conf->{$key});
5564             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5565
5566             foreach my $pci (@{$d->{pciid}}) {
5567                 my $pciid = $pci->{id};
5568                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5569             }
5570         }
5571
5572         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5573     };
5574     warn $@ if $@; # avoid errors - just warn
5575 }
5576
5577 # call only in locked context
5578 sub _do_vm_stop {
5579     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5580
5581     my $pid = check_running($vmid, $nocheck);
5582     return if !$pid;
5583
5584     my $conf;
5585     if (!$nocheck) {
5586         $conf = PVE::QemuConfig->load_config($vmid);
5587         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5588         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5589             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5590             $timeout = $opts->{down} if $opts->{down};
5591         }
5592         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5593     }
5594
5595     eval {
5596         if ($shutdown) {
5597             if (defined($conf) && get_qga_key($conf, 'enabled')) {
5598                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5599             } else {
5600                 mon_cmd($vmid, "system_powerdown");
5601             }
5602         } else {
5603             mon_cmd($vmid, "quit");
5604         }
5605     };
5606     my $err = $@;
5607
5608     if (!$err) {
5609         $timeout = 60 if !defined($timeout);
5610
5611         my $count = 0;
5612         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5613             $count++;
5614             sleep 1;
5615         }
5616
5617         if ($count >= $timeout) {
5618             if ($force) {
5619                 warn "VM still running - terminating now with SIGTERM\n";
5620                 kill 15, $pid;
5621             } else {
5622                 die "VM quit/powerdown failed - got timeout\n";
5623             }
5624         } else {
5625             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5626             return;
5627         }
5628     } else {
5629        if (!check_running($vmid, $nocheck)) {
5630             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5631             return;
5632        }
5633        if ($force) {
5634             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5635             kill 15, $pid;
5636         } else {
5637             die "VM quit/powerdown failed\n";
5638         }
5639     }
5640
5641     # wait again
5642     $timeout = 10;
5643
5644     my $count = 0;
5645     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5646         $count++;
5647         sleep 1;
5648     }
5649
5650     if ($count >= $timeout) {
5651         warn "VM still running - terminating now with SIGKILL\n";
5652         kill 9, $pid;
5653         sleep 1;
5654     }
5655
5656     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5657 }
5658
5659 # Note: use $nocheck to skip tests if VM configuration file exists.
5660 # We need that when migration VMs to other nodes (files already moved)
5661 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5662 sub vm_stop {
5663     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5664
5665     $force = 1 if !defined($force) && !$shutdown;
5666
5667     if ($migratedfrom){
5668         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5669         kill 15, $pid if $pid;
5670         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5671         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5672         return;
5673     }
5674
5675     PVE::QemuConfig->lock_config($vmid, sub {
5676         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5677    });
5678 }
5679
5680 sub vm_reboot {
5681     my ($vmid, $timeout) = @_;
5682
5683     PVE::QemuConfig->lock_config($vmid, sub {
5684         eval {
5685
5686             # only reboot if running, as qmeventd starts it again on a stop event
5687             return if !check_running($vmid);
5688
5689             create_reboot_request($vmid);
5690
5691             my $storecfg = PVE::Storage::config();
5692             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5693
5694         };
5695         if (my $err = $@) {
5696             # avoid that the next normal shutdown will be confused for a reboot
5697             clear_reboot_request($vmid);
5698             die $err;
5699         }
5700    });
5701 }
5702
5703 # note: if using the statestorage parameter, the caller has to check privileges
5704 sub vm_suspend {
5705     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5706
5707     my $conf;
5708     my $path;
5709     my $storecfg;
5710     my $vmstate;
5711
5712     PVE::QemuConfig->lock_config($vmid, sub {
5713
5714         $conf = PVE::QemuConfig->load_config($vmid);
5715
5716         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5717         PVE::QemuConfig->check_lock($conf)
5718             if !($skiplock || $is_backing_up);
5719
5720         die "cannot suspend to disk during backup\n"
5721             if $is_backing_up && $includestate;
5722
5723         if ($includestate) {
5724             $conf->{lock} = 'suspending';
5725             my $date = strftime("%Y-%m-%d", localtime(time()));
5726             $storecfg = PVE::Storage::config();
5727             if (!$statestorage) {
5728                 $statestorage = find_vmstate_storage($conf, $storecfg);
5729                 # check permissions for the storage
5730                 my $rpcenv = PVE::RPCEnvironment::get();
5731                 if ($rpcenv->{type} ne 'cli') {
5732                     my $authuser = $rpcenv->get_user();
5733                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5734                 }
5735             }
5736
5737
5738             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
5739                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5740             $path = PVE::Storage::path($storecfg, $vmstate);
5741             PVE::QemuConfig->write_config($vmid, $conf);
5742         } else {
5743             mon_cmd($vmid, "stop");
5744         }
5745     });
5746
5747     if ($includestate) {
5748         # save vm state
5749         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5750
5751         eval {
5752             set_migration_caps($vmid, 1);
5753             mon_cmd($vmid, "savevm-start", statefile => $path);
5754             for(;;) {
5755                 my $state = mon_cmd($vmid, "query-savevm");
5756                 if (!$state->{status}) {
5757                     die "savevm not active\n";
5758                 } elsif ($state->{status} eq 'active') {
5759                     sleep(1);
5760                     next;
5761                 } elsif ($state->{status} eq 'completed') {
5762                     print "State saved, quitting\n";
5763                     last;
5764                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5765                     die "query-savevm failed with error '$state->{error}'\n"
5766                 } else {
5767                     die "query-savevm returned status '$state->{status}'\n";
5768                 }
5769             }
5770         };
5771         my $err = $@;
5772
5773         PVE::QemuConfig->lock_config($vmid, sub {
5774             $conf = PVE::QemuConfig->load_config($vmid);
5775             if ($err) {
5776                 # cleanup, but leave suspending lock, to indicate something went wrong
5777                 eval {
5778                     mon_cmd($vmid, "savevm-end");
5779                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5780                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5781                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
5782                     PVE::QemuConfig->write_config($vmid, $conf);
5783                 };
5784                 warn $@ if $@;
5785                 die $err;
5786             }
5787
5788             die "lock changed unexpectedly\n"
5789                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5790
5791             mon_cmd($vmid, "quit");
5792             $conf->{lock} = 'suspended';
5793             PVE::QemuConfig->write_config($vmid, $conf);
5794         });
5795     }
5796 }
5797
5798 sub vm_resume {
5799     my ($vmid, $skiplock, $nocheck) = @_;
5800
5801     PVE::QemuConfig->lock_config($vmid, sub {
5802         my $res = mon_cmd($vmid, 'query-status');
5803         my $resume_cmd = 'cont';
5804         my $reset = 0;
5805
5806         if ($res->{status}) {
5807             return if $res->{status} eq 'running'; # job done, go home
5808             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
5809             $reset = 1 if $res->{status} eq 'shutdown';
5810         }
5811
5812         if (!$nocheck) {
5813
5814             my $conf = PVE::QemuConfig->load_config($vmid);
5815
5816             PVE::QemuConfig->check_lock($conf)
5817                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5818         }
5819
5820         if ($reset) {
5821             # required if a VM shuts down during a backup and we get a resume
5822             # request before the backup finishes for example
5823             mon_cmd($vmid, "system_reset");
5824         }
5825         mon_cmd($vmid, $resume_cmd);
5826     });
5827 }
5828
5829 sub vm_sendkey {
5830     my ($vmid, $skiplock, $key) = @_;
5831
5832     PVE::QemuConfig->lock_config($vmid, sub {
5833
5834         my $conf = PVE::QemuConfig->load_config($vmid);
5835
5836         # there is no qmp command, so we use the human monitor command
5837         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5838         die $res if $res ne '';
5839     });
5840 }
5841
5842 # vzdump restore implementaion
5843
5844 sub tar_archive_read_firstfile {
5845     my $archive = shift;
5846
5847     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5848
5849     # try to detect archive type first
5850     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5851         die "unable to open file '$archive'\n";
5852     my $firstfile = <$fh>;
5853     kill 15, $pid;
5854     close $fh;
5855
5856     die "ERROR: archive contaions no data\n" if !$firstfile;
5857     chomp $firstfile;
5858
5859     return $firstfile;
5860 }
5861
5862 sub tar_restore_cleanup {
5863     my ($storecfg, $statfile) = @_;
5864
5865     print STDERR "starting cleanup\n";
5866
5867     if (my $fd = IO::File->new($statfile, "r")) {
5868         while (defined(my $line = <$fd>)) {
5869             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5870                 my $volid = $2;
5871                 eval {
5872                     if ($volid =~ m|^/|) {
5873                         unlink $volid || die 'unlink failed\n';
5874                     } else {
5875                         PVE::Storage::vdisk_free($storecfg, $volid);
5876                     }
5877                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5878                 };
5879                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5880             } else {
5881                 print STDERR "unable to parse line in statfile - $line";
5882             }
5883         }
5884         $fd->close();
5885     }
5886 }
5887
5888 sub restore_file_archive {
5889     my ($archive, $vmid, $user, $opts) = @_;
5890
5891     return restore_vma_archive($archive, $vmid, $user, $opts)
5892         if $archive eq '-';
5893
5894     my $info = PVE::Storage::archive_info($archive);
5895     my $format = $opts->{format} // $info->{format};
5896     my $comp = $info->{compression};
5897
5898     # try to detect archive format
5899     if ($format eq 'tar') {
5900         return restore_tar_archive($archive, $vmid, $user, $opts);
5901     } else {
5902         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5903     }
5904 }
5905
5906 # hepler to remove disks that will not be used after restore
5907 my $restore_cleanup_oldconf = sub {
5908     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
5909
5910     PVE::QemuConfig->foreach_volume($oldconf, sub {
5911         my ($ds, $drive) = @_;
5912
5913         return if drive_is_cdrom($drive, 1);
5914
5915         my $volid = $drive->{file};
5916         return if !$volid || $volid =~ m|^/|;
5917
5918         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
5919         return if !$path || !$owner || ($owner != $vmid);
5920
5921         # Note: only delete disk we want to restore
5922         # other volumes will become unused
5923         if ($virtdev_hash->{$ds}) {
5924             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
5925             if (my $err = $@) {
5926                 warn $err;
5927             }
5928         }
5929     });
5930
5931     # delete vmstate files, after the restore we have no snapshots anymore
5932     foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5933         my $snap = $oldconf->{snapshots}->{$snapname};
5934         if ($snap->{vmstate}) {
5935             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
5936             if (my $err = $@) {
5937                 warn $err;
5938             }
5939         }
5940     }
5941 };
5942
5943 # Helper to parse vzdump backup device hints
5944 #
5945 # $rpcenv: Environment, used to ckeck storage permissions
5946 # $user: User ID, to check storage permissions
5947 # $storecfg: Storage configuration
5948 # $fh: the file handle for reading the configuration
5949 # $devinfo: should contain device sizes for all backu-up'ed devices
5950 # $options: backup options (pool, default storage)
5951 #
5952 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
5953 my $parse_backup_hints = sub {
5954     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
5955
5956     my $virtdev_hash = {};
5957
5958     while (defined(my $line = <$fh>)) {
5959         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5960             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5961             die "archive does not contain data for drive '$virtdev'\n"
5962                 if !$devinfo->{$devname};
5963
5964             if (defined($options->{storage})) {
5965                 $storeid = $options->{storage} || 'local';
5966             } elsif (!$storeid) {
5967                 $storeid = 'local';
5968             }
5969             $format = 'raw' if !$format;
5970             $devinfo->{$devname}->{devname} = $devname;
5971             $devinfo->{$devname}->{virtdev} = $virtdev;
5972             $devinfo->{$devname}->{format} = $format;
5973             $devinfo->{$devname}->{storeid} = $storeid;
5974
5975             # check permission on storage
5976             my $pool = $options->{pool}; # todo: do we need that?
5977             if ($user ne 'root@pam') {
5978                 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5979             }
5980
5981             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5982         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
5983             my $virtdev = $1;
5984             my $drive = parse_drive($virtdev, $2);
5985             if (drive_is_cloudinit($drive)) {
5986                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5987                 $storeid = $options->{storage} if defined ($options->{storage});
5988                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5989                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
5990
5991                 $virtdev_hash->{$virtdev} = {
5992                     format => $format,
5993                     storeid => $storeid,
5994                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
5995                     is_cloudinit => 1,
5996                 };
5997             }
5998         }
5999     }
6000
6001     return $virtdev_hash;
6002 };
6003
6004 # Helper to allocate and activate all volumes required for a restore
6005 #
6006 # $storecfg: Storage configuration
6007 # $virtdev_hash: as returned by parse_backup_hints()
6008 #
6009 # Returns: { $virtdev => $volid }
6010 my $restore_allocate_devices = sub {
6011     my ($storecfg, $virtdev_hash, $vmid) = @_;
6012
6013     my $map = {};
6014     foreach my $virtdev (sort keys %$virtdev_hash) {
6015         my $d = $virtdev_hash->{$virtdev};
6016         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6017         my $storeid = $d->{storeid};
6018         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6019
6020         # test if requested format is supported
6021         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6022         my $supported = grep { $_ eq $d->{format} } @$validFormats;
6023         $d->{format} = $defFormat if !$supported;
6024
6025         my $name;
6026         if ($d->{is_cloudinit}) {
6027             $name = "vm-$vmid-cloudinit";
6028             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6029             if ($scfg->{path}) {
6030                 $name .= ".$d->{format}";
6031             }
6032         }
6033
6034         my $volid = PVE::Storage::vdisk_alloc(
6035             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6036
6037         print STDERR "new volume ID is '$volid'\n";
6038         $d->{volid} = $volid;
6039
6040         PVE::Storage::activate_volumes($storecfg, [$volid]);
6041
6042         $map->{$virtdev} = $volid;
6043     }
6044
6045     return $map;
6046 };
6047
6048 sub restore_update_config_line {
6049     my ($cookie, $map, $line, $unique) = @_;
6050
6051     return '' if $line =~ m/^\#qmdump\#/;
6052     return '' if $line =~ m/^\#vzdump\#/;
6053     return '' if $line =~ m/^lock:/;
6054     return '' if $line =~ m/^unused\d+:/;
6055     return '' if $line =~ m/^parent:/;
6056
6057     my $res = '';
6058
6059     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6060     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6061         # try to convert old 1.X settings
6062         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6063         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6064             my ($model, $macaddr) = split(/\=/, $devconfig);
6065             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6066             my $net = {
6067                 model => $model,
6068                 bridge => "vmbr$ind",
6069                 macaddr => $macaddr,
6070             };
6071             my $netstr = print_net($net);
6072
6073             $res .= "net$cookie->{netcount}: $netstr\n";
6074             $cookie->{netcount}++;
6075         }
6076     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6077         my ($id, $netstr) = ($1, $2);
6078         my $net = parse_net($netstr);
6079         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6080         $netstr = print_net($net);
6081         $res .= "$id: $netstr\n";
6082     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
6083         my $virtdev = $1;
6084         my $value = $3;
6085         my $di = parse_drive($virtdev, $value);
6086         if (defined($di->{backup}) && !$di->{backup}) {
6087             $res .= "#$line";
6088         } elsif ($map->{$virtdev}) {
6089             delete $di->{format}; # format can change on restore
6090             $di->{file} = $map->{$virtdev};
6091             $value = print_drive($di);
6092             $res .= "$virtdev: $value\n";
6093         } else {
6094             $res .= $line;
6095         }
6096     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6097         my $vmgenid = $1;
6098         if ($vmgenid ne '0') {
6099             # always generate a new vmgenid if there was a valid one setup
6100             $vmgenid = generate_uuid();
6101         }
6102         $res .= "vmgenid: $vmgenid\n";
6103     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6104         my ($uuid, $uuid_str);
6105         UUID::generate($uuid);
6106         UUID::unparse($uuid, $uuid_str);
6107         my $smbios1 = parse_smbios1($2);
6108         $smbios1->{uuid} = $uuid_str;
6109         $res .= $1.print_smbios1($smbios1)."\n";
6110     } else {
6111         $res .= $line;
6112     }
6113
6114     return $res;
6115 }
6116
6117 my $restore_deactivate_volumes = sub {
6118     my ($storecfg, $devinfo) = @_;
6119
6120     my $vollist = [];
6121     foreach my $devname (keys %$devinfo) {
6122         my $volid = $devinfo->{$devname}->{volid};
6123         push @$vollist, $volid if $volid;
6124     }
6125
6126     PVE::Storage::deactivate_volumes($storecfg, $vollist);
6127 };
6128
6129 my $restore_destroy_volumes = sub {
6130     my ($storecfg, $devinfo) = @_;
6131
6132     foreach my $devname (keys %$devinfo) {
6133         my $volid = $devinfo->{$devname}->{volid};
6134         next if !$volid;
6135         eval {
6136             if ($volid =~ m|^/|) {
6137                 unlink $volid || die 'unlink failed\n';
6138             } else {
6139                 PVE::Storage::vdisk_free($storecfg, $volid);
6140             }
6141             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6142         };
6143         print STDERR "unable to cleanup '$volid' - $@" if $@;
6144     }
6145 };
6146
6147 sub scan_volids {
6148     my ($cfg, $vmid) = @_;
6149
6150     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6151
6152     my $volid_hash = {};
6153     foreach my $storeid (keys %$info) {
6154         foreach my $item (@{$info->{$storeid}}) {
6155             next if !($item->{volid} && $item->{size});
6156             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6157             $volid_hash->{$item->{volid}} = $item;
6158         }
6159     }
6160
6161     return $volid_hash;
6162 }
6163
6164 sub update_disk_config {
6165     my ($vmid, $conf, $volid_hash) = @_;
6166
6167     my $changes;
6168     my $prefix = "VM $vmid";
6169
6170     # used and unused disks
6171     my $referenced = {};
6172
6173     # Note: it is allowed to define multiple storages with same path (alias), so
6174     # we need to check both 'volid' and real 'path' (two different volid can point
6175     # to the same path).
6176
6177     my $referencedpath = {};
6178
6179     # update size info
6180     PVE::QemuConfig->foreach_volume($conf, sub {
6181         my ($opt, $drive) = @_;
6182
6183         my $volid = $drive->{file};
6184         return if !$volid;
6185         my $volume = $volid_hash->{$volid};
6186
6187         # mark volid as "in-use" for next step
6188         $referenced->{$volid} = 1;
6189         if ($volume && (my $path = $volume->{path})) {
6190             $referencedpath->{$path} = 1;
6191         }
6192
6193         return if drive_is_cdrom($drive);
6194         return if !$volume;
6195
6196         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6197         if (defined($updated)) {
6198             $changes = 1;
6199             $conf->{$opt} = print_drive($updated);
6200             print "$prefix ($opt): $msg\n";
6201         }
6202     });
6203
6204     # remove 'unusedX' entry if volume is used
6205     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6206         my ($opt, $drive) = @_;
6207
6208         my $volid = $drive->{file};
6209         return if !$volid;
6210
6211         my $path;
6212         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6213         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6214             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6215             $changes = 1;
6216             delete $conf->{$opt};
6217         }
6218
6219         $referenced->{$volid} = 1;
6220         $referencedpath->{$path} = 1 if $path;
6221     });
6222
6223     foreach my $volid (sort keys %$volid_hash) {
6224         next if $volid =~ m/vm-$vmid-state-/;
6225         next if $referenced->{$volid};
6226         my $path = $volid_hash->{$volid}->{path};
6227         next if !$path; # just to be sure
6228         next if $referencedpath->{$path};
6229         $changes = 1;
6230         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6231         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6232         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6233     }
6234
6235     return $changes;
6236 }
6237
6238 sub rescan {
6239     my ($vmid, $nolock, $dryrun) = @_;
6240
6241     my $cfg = PVE::Storage::config();
6242
6243     print "rescan volumes...\n";
6244     my $volid_hash = scan_volids($cfg, $vmid);
6245
6246     my $updatefn =  sub {
6247         my ($vmid) = @_;
6248
6249         my $conf = PVE::QemuConfig->load_config($vmid);
6250
6251         PVE::QemuConfig->check_lock($conf);
6252
6253         my $vm_volids = {};
6254         foreach my $volid (keys %$volid_hash) {
6255             my $info = $volid_hash->{$volid};
6256             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6257         }
6258
6259         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6260
6261         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6262     };
6263
6264     if (defined($vmid)) {
6265         if ($nolock) {
6266             &$updatefn($vmid);
6267         } else {
6268             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6269         }
6270     } else {
6271         my $vmlist = config_list();
6272         foreach my $vmid (keys %$vmlist) {
6273             if ($nolock) {
6274                 &$updatefn($vmid);
6275             } else {
6276                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6277             }
6278         }
6279     }
6280 }
6281
6282 sub restore_proxmox_backup_archive {
6283     my ($archive, $vmid, $user, $options) = @_;
6284
6285     my $storecfg = PVE::Storage::config();
6286
6287     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6288     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6289
6290     my $fingerprint = $scfg->{fingerprint};
6291     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6292
6293     my $repo = PVE::PBSClient::get_repository($scfg);
6294
6295     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6296     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6297     local $ENV{PBS_PASSWORD} = $password;
6298     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6299
6300     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6301         PVE::Storage::parse_volname($storecfg, $archive);
6302
6303     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6304
6305     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6306
6307     my $tmpdir = "/var/tmp/vzdumptmp$$";
6308     rmtree $tmpdir;
6309     mkpath $tmpdir;
6310
6311     my $conffile = PVE::QemuConfig->config_file($vmid);
6312      # disable interrupts (always do cleanups)
6313     local $SIG{INT} =
6314         local $SIG{TERM} =
6315         local $SIG{QUIT} =
6316         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6317
6318     # Note: $oldconf is undef if VM does not exists
6319     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6320     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6321     my $new_conf_raw = '';
6322
6323     my $rpcenv = PVE::RPCEnvironment::get();
6324     my $devinfo = {};
6325
6326     eval {
6327         # enable interrupts
6328         local $SIG{INT} =
6329             local $SIG{TERM} =
6330             local $SIG{QUIT} =
6331             local $SIG{HUP} =
6332             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6333
6334         my $cfgfn = "$tmpdir/qemu-server.conf";
6335         my $firewall_config_fn = "$tmpdir/fw.conf";
6336         my $index_fn = "$tmpdir/index.json";
6337
6338         my $cmd = "restore";
6339
6340         my $param = [$pbs_backup_name, "index.json", $index_fn];
6341         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6342         my $index = PVE::Tools::file_get_contents($index_fn);
6343         $index = decode_json($index);
6344
6345         # print Dumper($index);
6346         foreach my $info (@{$index->{files}}) {
6347             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6348                 my $devname = $1;
6349                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6350                     $devinfo->{$devname}->{size} = $1;
6351                 } else {
6352                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6353                 }
6354             }
6355         }
6356
6357         my $is_qemu_server_backup = scalar(
6358             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6359         );
6360         if (!$is_qemu_server_backup) {
6361             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6362         }
6363         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6364
6365         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6366         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6367
6368         if ($has_firewall_config) {
6369             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6370             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6371
6372             my $pve_firewall_dir = '/etc/pve/firewall';
6373             mkdir $pve_firewall_dir; # make sure the dir exists
6374             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6375         }
6376
6377         my $fh = IO::File->new($cfgfn, "r") ||
6378             die "unable to read qemu-server.conf - $!\n";
6379
6380         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6381
6382         # fixme: rate limit?
6383
6384         # create empty/temp config
6385         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6386
6387         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6388
6389         # allocate volumes
6390         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6391
6392         foreach my $virtdev (sort keys %$virtdev_hash) {
6393             my $d = $virtdev_hash->{$virtdev};
6394             next if $d->{is_cloudinit}; # no need to restore cloudinit
6395
6396             # this fails if storage is unavailable
6397             my $volid = $d->{volid};
6398             my $path = PVE::Storage::path($storecfg, $volid);
6399
6400             # for live-restore we only want to preload the efidisk
6401             next if $options->{live} && $virtdev ne 'efidisk0';
6402
6403             my $pbs_restore_cmd = [
6404                 '/usr/bin/pbs-restore',
6405                 '--repository', $repo,
6406                 $pbs_backup_name,
6407                 "$d->{devname}.img.fidx",
6408                 $path,
6409                 '--verbose',
6410                 ];
6411
6412             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6413             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
6414
6415             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6416                 push @$pbs_restore_cmd, '--skip-zero';
6417             }
6418
6419             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6420             print "restore proxmox backup image: $dbg_cmdstring\n";
6421             run_command($pbs_restore_cmd);
6422         }
6423
6424         $fh->seek(0, 0) || die "seek failed - $!\n";
6425
6426         my $cookie = { netcount => 0 };
6427         while (defined(my $line = <$fh>)) {
6428             $new_conf_raw .= restore_update_config_line(
6429                 $cookie,
6430                 $map,
6431                 $line,
6432                 $options->{unique},
6433             );
6434         }
6435
6436         $fh->close();
6437     };
6438     my $err = $@;
6439
6440     if ($err || !$options->{live}) {
6441         $restore_deactivate_volumes->($storecfg, $devinfo);
6442     }
6443
6444     rmtree $tmpdir;
6445
6446     if ($err) {
6447         $restore_destroy_volumes->($storecfg, $devinfo);
6448         die $err;
6449     }
6450
6451     if ($options->{live}) {
6452         # keep lock during live-restore
6453         $new_conf_raw .= "\nlock: create";
6454     }
6455
6456     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6457
6458     PVE::Cluster::cfs_update(); # make sure we read new file
6459
6460     eval { rescan($vmid, 1); };
6461     warn $@ if $@;
6462
6463     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
6464
6465     if ($options->{live}) {
6466         # enable interrupts
6467         local $SIG{INT} =
6468             local $SIG{TERM} =
6469             local $SIG{QUIT} =
6470             local $SIG{HUP} =
6471             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
6472
6473         my $conf = PVE::QemuConfig->load_config($vmid);
6474         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
6475
6476         delete $devinfo->{'drive-efidisk0'}; # this special drive is already restored before start
6477         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $repo, $keyfile, $pbs_backup_name);
6478
6479         PVE::QemuConfig->remove_lock($vmid, "create");
6480     }
6481 }
6482
6483 sub pbs_live_restore {
6484     my ($vmid, $conf, $storecfg, $restored_disks, $repo, $keyfile, $snap) = @_;
6485
6486     print "starting VM for live-restore\n";
6487     print "repository: '$repo', snapshot: '$snap'\n";
6488
6489     my $pbs_backing = {};
6490     for my $ds (keys %$restored_disks) {
6491         $ds =~ m/^drive-(.*)$/;
6492         my $confname = $1;
6493         $pbs_backing->{$confname} = {
6494             repository => $repo,
6495             snapshot => $snap,
6496             archive => "$ds.img.fidx",
6497         };
6498         $pbs_backing->{$confname}->{keyfile} = $keyfile if -e $keyfile;
6499
6500         my $drive = parse_drive($confname, $conf->{$confname});
6501         print "restoring '$ds' to '$drive->{file}'\n";
6502     }
6503
6504     my $drives_streamed = 0;
6505     eval {
6506         # make sure HA doesn't interrupt our restore by stopping the VM
6507         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
6508             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
6509         }
6510
6511         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
6512         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
6513         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
6514
6515         my $qmeventd_fd = register_qmeventd_handle($vmid);
6516
6517         # begin streaming, i.e. data copy from PBS to target disk for every vol,
6518         # this will effectively collapse the backing image chain consisting of
6519         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
6520         # removes itself once all backing images vanish with 'auto-remove=on')
6521         my $jobs = {};
6522         for my $ds (sort keys %$restored_disks) {
6523             my $job_id = "restore-$ds";
6524             mon_cmd($vmid, 'block-stream',
6525                 'job-id' => $job_id,
6526                 device => "$ds",
6527             );
6528             $jobs->{$job_id} = {};
6529         }
6530
6531         mon_cmd($vmid, 'cont');
6532         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
6533
6534         print "restore-drive jobs finished successfully, removing all tracking block devices"
6535             ." to disconnect from Proxmox Backup Server\n";
6536
6537         for my $ds (sort keys %$restored_disks) {
6538             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
6539         }
6540
6541         close($qmeventd_fd);
6542     };
6543
6544     my $err = $@;
6545
6546     if ($err) {
6547         warn "An error occured during live-restore: $err\n";
6548         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
6549         die "live-restore failed\n";
6550     }
6551 }
6552
6553 sub restore_vma_archive {
6554     my ($archive, $vmid, $user, $opts, $comp) = @_;
6555
6556     my $readfrom = $archive;
6557
6558     my $cfg = PVE::Storage::config();
6559     my $commands = [];
6560     my $bwlimit = $opts->{bwlimit};
6561
6562     my $dbg_cmdstring = '';
6563     my $add_pipe = sub {
6564         my ($cmd) = @_;
6565         push @$commands, $cmd;
6566         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6567         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6568         $readfrom = '-';
6569     };
6570
6571     my $input = undef;
6572     if ($archive eq '-') {
6573         $input = '<&STDIN';
6574     } else {
6575         # If we use a backup from a PVE defined storage we also consider that
6576         # storage's rate limit:
6577         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6578         if (defined($volid)) {
6579             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6580             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6581             if ($readlimit) {
6582                 print STDERR "applying read rate limit: $readlimit\n";
6583                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6584                 $add_pipe->($cstream);
6585             }
6586         }
6587     }
6588
6589     if ($comp) {
6590         my $info = PVE::Storage::decompressor_info('vma', $comp);
6591         my $cmd = $info->{decompressor};
6592         push @$cmd, $readfrom;
6593         $add_pipe->($cmd);
6594     }
6595
6596     my $tmpdir = "/var/tmp/vzdumptmp$$";
6597     rmtree $tmpdir;
6598
6599     # disable interrupts (always do cleanups)
6600     local $SIG{INT} =
6601         local $SIG{TERM} =
6602         local $SIG{QUIT} =
6603         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6604
6605     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6606     POSIX::mkfifo($mapfifo, 0600);
6607     my $fifofh;
6608     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
6609
6610     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6611
6612     my $oldtimeout;
6613     my $timeout = 5;
6614
6615     my $devinfo = {};
6616
6617     my $rpcenv = PVE::RPCEnvironment::get();
6618
6619     my $conffile = PVE::QemuConfig->config_file($vmid);
6620
6621     # Note: $oldconf is undef if VM does not exist
6622     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6623     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6624     my $new_conf_raw = '';
6625
6626     my %storage_limits;
6627
6628     my $print_devmap = sub {
6629         my $cfgfn = "$tmpdir/qemu-server.conf";
6630
6631         # we can read the config - that is already extracted
6632         my $fh = IO::File->new($cfgfn, "r") ||
6633             die "unable to read qemu-server.conf - $!\n";
6634
6635         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6636         if (-f $fwcfgfn) {
6637             my $pve_firewall_dir = '/etc/pve/firewall';
6638             mkdir $pve_firewall_dir; # make sure the dir exists
6639             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6640         }
6641
6642         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6643
6644         foreach my $info (values %{$virtdev_hash}) {
6645             my $storeid = $info->{storeid};
6646             next if defined($storage_limits{$storeid});
6647
6648             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
6649             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
6650             $storage_limits{$storeid} = $limit * 1024;
6651         }
6652
6653         foreach my $devname (keys %$devinfo) {
6654             die "found no device mapping information for device '$devname'\n"
6655                 if !$devinfo->{$devname}->{virtdev};
6656         }
6657
6658         # create empty/temp config
6659         if ($oldconf) {
6660             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6661             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6662         }
6663
6664         # allocate volumes
6665         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6666
6667         # print restore information to $fifofh
6668         foreach my $virtdev (sort keys %$virtdev_hash) {
6669             my $d = $virtdev_hash->{$virtdev};
6670             next if $d->{is_cloudinit}; # no need to restore cloudinit
6671
6672             my $storeid = $d->{storeid};
6673             my $volid = $d->{volid};
6674
6675             my $map_opts = '';
6676             if (my $limit = $storage_limits{$storeid}) {
6677                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6678             }
6679
6680             my $write_zeros = 1;
6681             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6682                 $write_zeros = 0;
6683             }
6684
6685             my $path = PVE::Storage::path($cfg, $volid);
6686
6687             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6688
6689             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6690         }
6691
6692         $fh->seek(0, 0) || die "seek failed - $!\n";
6693
6694         my $cookie = { netcount => 0 };
6695         while (defined(my $line = <$fh>)) {
6696             $new_conf_raw .= restore_update_config_line(
6697                 $cookie,
6698                 $map,
6699                 $line,
6700                 $opts->{unique},
6701             );
6702         }
6703
6704         $fh->close();
6705     };
6706
6707     eval {
6708         # enable interrupts
6709         local $SIG{INT} =
6710             local $SIG{TERM} =
6711             local $SIG{QUIT} =
6712             local $SIG{HUP} =
6713             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6714         local $SIG{ALRM} = sub { die "got timeout\n"; };
6715
6716         $oldtimeout = alarm($timeout);
6717
6718         my $parser = sub {
6719             my $line = shift;
6720
6721             print "$line\n";
6722
6723             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6724                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6725                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6726             } elsif ($line =~ m/^CTIME: /) {
6727                 # we correctly received the vma config, so we can disable
6728                 # the timeout now for disk allocation (set to 10 minutes, so
6729                 # that we always timeout if something goes wrong)
6730                 alarm(600);
6731                 &$print_devmap();
6732                 print $fifofh "done\n";
6733                 my $tmp = $oldtimeout || 0;
6734                 $oldtimeout = undef;
6735                 alarm($tmp);
6736                 close($fifofh);
6737                 $fifofh = undef;
6738             }
6739         };
6740
6741         print "restore vma archive: $dbg_cmdstring\n";
6742         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6743     };
6744     my $err = $@;
6745
6746     alarm($oldtimeout) if $oldtimeout;
6747
6748     $restore_deactivate_volumes->($cfg, $devinfo);
6749
6750     close($fifofh) if $fifofh;
6751     unlink $mapfifo;
6752     rmtree $tmpdir;
6753
6754     if ($err) {
6755         $restore_destroy_volumes->($cfg, $devinfo);
6756         die $err;
6757     }
6758
6759     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6760
6761     PVE::Cluster::cfs_update(); # make sure we read new file
6762
6763     eval { rescan($vmid, 1); };
6764     warn $@ if $@;
6765
6766     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
6767 }
6768
6769 sub restore_tar_archive {
6770     my ($archive, $vmid, $user, $opts) = @_;
6771
6772     if ($archive ne '-') {
6773         my $firstfile = tar_archive_read_firstfile($archive);
6774         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
6775             if $firstfile ne 'qemu-server.conf';
6776     }
6777
6778     my $storecfg = PVE::Storage::config();
6779
6780     # avoid zombie disks when restoring over an existing VM -> cleanup first
6781     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6782     # skiplock=1 because qmrestore has set the 'create' lock itself already
6783     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6784     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6785
6786     my $tocmd = "/usr/lib/qemu-server/qmextract";
6787
6788     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6789     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6790     $tocmd .= ' --prealloc' if $opts->{prealloc};
6791     $tocmd .= ' --info' if $opts->{info};
6792
6793     # tar option "xf" does not autodetect compression when read from STDIN,
6794     # so we pipe to zcat
6795     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6796         PVE::Tools::shellquote("--to-command=$tocmd");
6797
6798     my $tmpdir = "/var/tmp/vzdumptmp$$";
6799     mkpath $tmpdir;
6800
6801     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6802     local $ENV{VZDUMP_VMID} = $vmid;
6803     local $ENV{VZDUMP_USER} = $user;
6804
6805     my $conffile = PVE::QemuConfig->config_file($vmid);
6806     my $new_conf_raw = '';
6807
6808     # disable interrupts (always do cleanups)
6809     local $SIG{INT} =
6810         local $SIG{TERM} =
6811         local $SIG{QUIT} =
6812         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6813
6814     eval {
6815         # enable interrupts
6816         local $SIG{INT} =
6817             local $SIG{TERM} =
6818             local $SIG{QUIT} =
6819             local $SIG{HUP} =
6820             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6821
6822         if ($archive eq '-') {
6823             print "extracting archive from STDIN\n";
6824             run_command($cmd, input => "<&STDIN");
6825         } else {
6826             print "extracting archive '$archive'\n";
6827             run_command($cmd);
6828         }
6829
6830         return if $opts->{info};
6831
6832         # read new mapping
6833         my $map = {};
6834         my $statfile = "$tmpdir/qmrestore.stat";
6835         if (my $fd = IO::File->new($statfile, "r")) {
6836             while (defined (my $line = <$fd>)) {
6837                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6838                     $map->{$1} = $2 if $1;
6839                 } else {
6840                     print STDERR "unable to parse line in statfile - $line\n";
6841                 }
6842             }
6843             $fd->close();
6844         }
6845
6846         my $confsrc = "$tmpdir/qemu-server.conf";
6847
6848         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
6849
6850         my $cookie = { netcount => 0 };
6851         while (defined (my $line = <$srcfd>)) {
6852             $new_conf_raw .= restore_update_config_line(
6853                 $cookie,
6854                 $map,
6855                 $line,
6856                 $opts->{unique},
6857             );
6858         }
6859
6860         $srcfd->close();
6861     };
6862     if (my $err = $@) {
6863         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6864         die $err;
6865     }
6866
6867     rmtree $tmpdir;
6868
6869     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6870
6871     PVE::Cluster::cfs_update(); # make sure we read new file
6872
6873     eval { rescan($vmid, 1); };
6874     warn $@ if $@;
6875 };
6876
6877 sub foreach_storage_used_by_vm {
6878     my ($conf, $func) = @_;
6879
6880     my $sidhash = {};
6881
6882     PVE::QemuConfig->foreach_volume($conf, sub {
6883         my ($ds, $drive) = @_;
6884         return if drive_is_cdrom($drive);
6885
6886         my $volid = $drive->{file};
6887
6888         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6889         $sidhash->{$sid} = $sid if $sid;
6890     });
6891
6892     foreach my $sid (sort keys %$sidhash) {
6893         &$func($sid);
6894     }
6895 }
6896
6897 my $qemu_snap_storage = {
6898     rbd => 1,
6899 };
6900 sub do_snapshots_with_qemu {
6901     my ($storecfg, $volid) = @_;
6902
6903     my $storage_name = PVE::Storage::parse_volume_id($volid);
6904     my $scfg = $storecfg->{ids}->{$storage_name};
6905     die "could not find storage '$storage_name'\n" if !defined($scfg);
6906
6907     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6908         return 1;
6909     }
6910
6911     if ($volid =~ m/\.(qcow2|qed)$/){
6912         return 1;
6913     }
6914
6915     return;
6916 }
6917
6918 sub qga_check_running {
6919     my ($vmid, $nowarn) = @_;
6920
6921     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6922     if ($@) {
6923         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6924         return 0;
6925     }
6926     return 1;
6927 }
6928
6929 sub template_create {
6930     my ($vmid, $conf, $disk) = @_;
6931
6932     my $storecfg = PVE::Storage::config();
6933
6934     PVE::QemuConfig->foreach_volume($conf, sub {
6935         my ($ds, $drive) = @_;
6936
6937         return if drive_is_cdrom($drive);
6938         return if $disk && $ds ne $disk;
6939
6940         my $volid = $drive->{file};
6941         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6942
6943         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6944         $drive->{file} = $voliddst;
6945         $conf->{$ds} = print_drive($drive);
6946         PVE::QemuConfig->write_config($vmid, $conf);
6947     });
6948 }
6949
6950 sub convert_iscsi_path {
6951     my ($path) = @_;
6952
6953     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6954         my $portal = $1;
6955         my $target = $2;
6956         my $lun = $3;
6957
6958         my $initiator_name = get_initiator_name();
6959
6960         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6961                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6962     }
6963
6964     die "cannot convert iscsi path '$path', unkown format\n";
6965 }
6966
6967 sub qemu_img_convert {
6968     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6969
6970     my $storecfg = PVE::Storage::config();
6971     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6972     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6973
6974     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6975
6976     my $cachemode;
6977     my $src_path;
6978     my $src_is_iscsi = 0;
6979     my $src_format;
6980
6981     if ($src_storeid) {
6982         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6983         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6984         $src_format = qemu_img_format($src_scfg, $src_volname);
6985         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6986         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6987         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6988     } elsif (-f $src_volid) {
6989         $src_path = $src_volid;
6990         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6991             $src_format = $1;
6992         }
6993     }
6994
6995     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6996
6997     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6998     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6999     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7000     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
7001
7002     my $cmd = [];
7003     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7004     push @$cmd, '-l', "snapshot.name=$snapname"
7005         if $snapname && $src_format && $src_format eq "qcow2";
7006     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
7007     push @$cmd, '-T', $cachemode if defined($cachemode);
7008
7009     if ($src_is_iscsi) {
7010         push @$cmd, '--image-opts';
7011         $src_path = convert_iscsi_path($src_path);
7012     } elsif ($src_format) {
7013         push @$cmd, '-f', $src_format;
7014     }
7015
7016     if ($dst_is_iscsi) {
7017         push @$cmd, '--target-image-opts';
7018         $dst_path = convert_iscsi_path($dst_path);
7019     } else {
7020         push @$cmd, '-O', $dst_format;
7021     }
7022
7023     push @$cmd, $src_path;
7024
7025     if (!$dst_is_iscsi && $is_zero_initialized) {
7026         push @$cmd, "zeroinit:$dst_path";
7027     } else {
7028         push @$cmd, $dst_path;
7029     }
7030
7031     my $parser = sub {
7032         my $line = shift;
7033         if($line =~ m/\((\S+)\/100\%\)/){
7034             my $percent = $1;
7035             my $transferred = int($size * $percent / 100);
7036             my $total_h = render_bytes($size, 1);
7037             my $transferred_h = render_bytes($transferred, 1);
7038
7039             print "transferred $transferred_h of $total_h ($percent%)\n";
7040         }
7041
7042     };
7043
7044     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7045     my $err = $@;
7046     die "copy failed: $err" if $err;
7047 }
7048
7049 sub qemu_img_format {
7050     my ($scfg, $volname) = @_;
7051
7052     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7053         return $1;
7054     } else {
7055         return "raw";
7056     }
7057 }
7058
7059 sub qemu_drive_mirror {
7060     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7061
7062     $jobs = {} if !$jobs;
7063
7064     my $qemu_target;
7065     my $format;
7066     $jobs->{"drive-$drive"} = {};
7067
7068     if ($dst_volid =~ /^nbd:/) {
7069         $qemu_target = $dst_volid;
7070         $format = "nbd";
7071     } else {
7072         my $storecfg = PVE::Storage::config();
7073         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7074
7075         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7076
7077         $format = qemu_img_format($dst_scfg, $dst_volname);
7078
7079         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7080
7081         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7082     }
7083
7084     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7085     $opts->{format} = $format if $format;
7086
7087     if (defined($src_bitmap)) {
7088         $opts->{sync} = 'incremental';
7089         $opts->{bitmap} = $src_bitmap;
7090         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7091     }
7092
7093     if (defined($bwlimit)) {
7094         $opts->{speed} = $bwlimit * 1024;
7095         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7096     } else {
7097         print "drive mirror is starting for drive-$drive\n";
7098     }
7099
7100     # if a job already runs for this device we get an error, catch it for cleanup
7101     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7102     if (my $err = $@) {
7103         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7104         warn "$@\n" if $@;
7105         die "mirroring error: $err\n";
7106     }
7107
7108     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7109 }
7110
7111 # $completion can be either
7112 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7113 # 'cancel': wait until all jobs are ready, block-job-cancel them
7114 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7115 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7116 sub qemu_drive_mirror_monitor {
7117     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7118
7119     $completion //= 'complete';
7120     $op //= "mirror";
7121
7122     eval {
7123         my $err_complete = 0;
7124
7125         my $starttime = time ();
7126         while (1) {
7127             die "block job ('$op') timed out\n" if $err_complete > 300;
7128
7129             my $stats = mon_cmd($vmid, "query-block-jobs");
7130             my $ctime = time();
7131
7132             my $running_jobs = {};
7133             for my $stat (@$stats) {
7134                 next if $stat->{type} ne $op;
7135                 $running_jobs->{$stat->{device}} = $stat;
7136             }
7137
7138             my $readycounter = 0;
7139
7140             for my $job_id (sort keys %$jobs) {
7141                 my $job = $running_jobs->{$job_id};
7142
7143                 my $vanished = !defined($job);
7144                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7145                 if($complete || ($vanished && $completion eq 'auto')) {
7146                     print "$job_id: $op-job finished\n";
7147                     delete $jobs->{$job_id};
7148                     next;
7149                 }
7150
7151                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7152
7153                 my $busy = $job->{busy};
7154                 my $ready = $job->{ready};
7155                 if (my $total = $job->{len}) {
7156                     my $transferred = $job->{offset} || 0;
7157                     my $remaining = $total - $transferred;
7158                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7159
7160                     my $duration = $ctime - $starttime;
7161                     my $total_h = render_bytes($total, 1);
7162                     my $transferred_h = render_bytes($transferred, 1);
7163
7164                     my $status = sprintf(
7165                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7166                         render_duration($duration),
7167                     );
7168
7169                     if ($ready) {
7170                         if ($busy) {
7171                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7172                         } else {
7173                             $status .= ", ready";
7174                         }
7175                     }
7176                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7177                     $jobs->{$job_id}->{ready} = $ready;
7178                 }
7179
7180                 $readycounter++ if $job->{ready};
7181             }
7182
7183             last if scalar(keys %$jobs) == 0;
7184
7185             if ($readycounter == scalar(keys %$jobs)) {
7186                 print "all '$op' jobs are ready\n";
7187
7188                 # do the complete later (or has already been done)
7189                 last if $completion eq 'skip' || $completion eq 'auto';
7190
7191                 if ($vmiddst && $vmiddst != $vmid) {
7192                     my $agent_running = $qga && qga_check_running($vmid);
7193                     if ($agent_running) {
7194                         print "freeze filesystem\n";
7195                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7196                     } else {
7197                         print "suspend vm\n";
7198                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7199                     }
7200
7201                     # if we clone a disk for a new target vm, we don't switch the disk
7202                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7203
7204                     if ($agent_running) {
7205                         print "unfreeze filesystem\n";
7206                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7207                     } else {
7208                         print "resume vm\n";
7209                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
7210                     }
7211
7212                     last;
7213                 } else {
7214
7215                     for my $job_id (sort keys %$jobs) {
7216                         # try to switch the disk if source and destination are on the same guest
7217                         print "$job_id: Completing block job_id...\n";
7218
7219                         my $op;
7220                         if ($completion eq 'complete') {
7221                             $op = 'block-job-complete';
7222                         } elsif ($completion eq 'cancel') {
7223                             $op = 'block-job-cancel';
7224                         } else {
7225                             die "invalid completion value: $completion\n";
7226                         }
7227                         eval { mon_cmd($vmid, $op, device => $job_id) };
7228                         if ($@ =~ m/cannot be completed/) {
7229                             print "$job_id: block job cannot be completed, trying again.\n";
7230                             $err_complete++;
7231                         }else {
7232                             print "$job_id: Completed successfully.\n";
7233                             $jobs->{$job_id}->{complete} = 1;
7234                         }
7235                     }
7236                 }
7237             }
7238             sleep 1;
7239         }
7240     };
7241     my $err = $@;
7242
7243     if ($err) {
7244         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7245         die "block job ($op) error: $err";
7246     }
7247 }
7248
7249 sub qemu_blockjobs_cancel {
7250     my ($vmid, $jobs) = @_;
7251
7252     foreach my $job (keys %$jobs) {
7253         print "$job: Cancelling block job\n";
7254         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7255         $jobs->{$job}->{cancel} = 1;
7256     }
7257
7258     while (1) {
7259         my $stats = mon_cmd($vmid, "query-block-jobs");
7260
7261         my $running_jobs = {};
7262         foreach my $stat (@$stats) {
7263             $running_jobs->{$stat->{device}} = $stat;
7264         }
7265
7266         foreach my $job (keys %$jobs) {
7267
7268             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7269                 print "$job: Done.\n";
7270                 delete $jobs->{$job};
7271             }
7272         }
7273
7274         last if scalar(keys %$jobs) == 0;
7275
7276         sleep 1;
7277     }
7278 }
7279
7280 sub clone_disk {
7281     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7282         $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
7283
7284     my $newvolid;
7285
7286     if (!$full) {
7287         print "create linked clone of drive $drivename ($drive->{file})\n";
7288         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7289         push @$newvollist, $newvolid;
7290     } else {
7291
7292         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7293         $storeid = $storage if $storage;
7294
7295         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7296
7297         print "create full clone of drive $drivename ($drive->{file})\n";
7298         my $name = undef;
7299         my $size = undef;
7300         if (drive_is_cloudinit($drive)) {
7301             $name = "vm-$newvmid-cloudinit";
7302             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7303             if ($scfg->{path}) {
7304                 $name .= ".$dst_format";
7305             }
7306             $snapname = undef;
7307             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7308         } elsif ($drivename eq 'efidisk0') {
7309             $size = get_efivars_size($conf);
7310         } else {
7311             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
7312         }
7313         $newvolid = PVE::Storage::vdisk_alloc(
7314             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
7315         );
7316         push @$newvollist, $newvolid;
7317
7318         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7319
7320         if (drive_is_cloudinit($drive)) {
7321             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
7322             # if this is the case, we have to complete any block-jobs still there from
7323             # previous drive-mirrors
7324             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
7325                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
7326             }
7327             goto no_data_clone;
7328         }
7329
7330         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7331         if (!$running || $snapname) {
7332             # TODO: handle bwlimits
7333             if ($drivename eq 'efidisk0') {
7334                 # the relevant data on the efidisk may be smaller than the source
7335                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
7336                 # that is given by the OVMF_VARS.fd
7337                 my $src_path = PVE::Storage::path($storecfg, $drive->{file});
7338                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
7339
7340                 # better for Ceph if block size is not too small, see bug #3324
7341                 my $bs = 1024*1024;
7342
7343                 run_command(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=$bs", "osize=$size",
7344                     "if=$src_path", "of=$dst_path"]);
7345             } else {
7346                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7347             }
7348         } else {
7349
7350             my $kvmver = get_running_qemu_version ($vmid);
7351             if (!min_version($kvmver, 2, 7)) {
7352                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7353                     if $drive->{iothread};
7354             }
7355
7356             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7357                 $completion, $qga, $bwlimit);
7358         }
7359     }
7360
7361 no_data_clone:
7362     my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
7363
7364     my $disk = $drive;
7365     $disk->{format} = undef;
7366     $disk->{file} = $newvolid;
7367     $disk->{size} = $size if defined($size);
7368
7369     return $disk;
7370 }
7371
7372 sub get_running_qemu_version {
7373     my ($vmid) = @_;
7374     my $res = mon_cmd($vmid, "query-version");
7375     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7376 }
7377
7378 sub qemu_use_old_bios_files {
7379     my ($machine_type) = @_;
7380
7381     return if !$machine_type;
7382
7383     my $use_old_bios_files = undef;
7384
7385     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7386         $machine_type = $1;
7387         $use_old_bios_files = 1;
7388     } else {
7389         my $version = extract_version($machine_type, kvm_user_version());
7390         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7391         # load new efi bios files on migration. So this hack is required to allow
7392         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7393         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7394         $use_old_bios_files = !min_version($version, 2, 4);
7395     }
7396
7397     return ($use_old_bios_files, $machine_type);
7398 }
7399
7400 sub get_efivars_size {
7401     my ($conf) = @_;
7402     my $arch = get_vm_arch($conf);
7403     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7404     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
7405     return -s $ovmf_vars;
7406 }
7407
7408 sub update_efidisk_size {
7409     my ($conf) = @_;
7410
7411     return if !defined($conf->{efidisk0});
7412
7413     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
7414     $disk->{size} = get_efivars_size($conf);
7415     $conf->{efidisk0} = print_drive($disk);
7416
7417     return;
7418 }
7419
7420 sub create_efidisk($$$$$) {
7421     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7422
7423     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7424     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7425
7426     my $vars_size_b = -s $ovmf_vars;
7427     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7428     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7429     PVE::Storage::activate_volumes($storecfg, [$volid]);
7430
7431     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7432     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7433
7434     return ($volid, $size/1024);
7435 }
7436
7437 sub vm_iothreads_list {
7438     my ($vmid) = @_;
7439
7440     my $res = mon_cmd($vmid, 'query-iothreads');
7441
7442     my $iothreads = {};
7443     foreach my $iothread (@$res) {
7444         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7445     }
7446
7447     return $iothreads;
7448 }
7449
7450 sub scsihw_infos {
7451     my ($conf, $drive) = @_;
7452
7453     my $maxdev = 0;
7454
7455     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7456         $maxdev = 7;
7457     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7458         $maxdev = 1;
7459     } else {
7460         $maxdev = 256;
7461     }
7462
7463     my $controller = int($drive->{index} / $maxdev);
7464     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
7465         ? "virtioscsi"
7466         : "scsihw";
7467
7468     return ($maxdev, $controller, $controller_prefix);
7469 }
7470
7471 sub windows_version {
7472     my ($ostype) = @_;
7473
7474     return 0 if !$ostype;
7475
7476     my $winversion = 0;
7477
7478     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7479         $winversion = 5;
7480     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7481         $winversion = 6;
7482     } elsif ($ostype =~ m/^win(\d+)$/) {
7483         $winversion = $1;
7484     }
7485
7486     return $winversion;
7487 }
7488
7489 sub resolve_dst_disk_format {
7490         my ($storecfg, $storeid, $src_volname, $format) = @_;
7491         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7492
7493         if (!$format) {
7494             # if no target format is specified, use the source disk format as hint
7495             if ($src_volname) {
7496                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7497                 $format = qemu_img_format($scfg, $src_volname);
7498             } else {
7499                 return $defFormat;
7500             }
7501         }
7502
7503         # test if requested format is supported - else use default
7504         my $supported = grep { $_ eq $format } @$validFormats;
7505         $format = $defFormat if !$supported;
7506         return $format;
7507 }
7508
7509 # NOTE: if this logic changes, please update docs & possibly gui logic
7510 sub find_vmstate_storage {
7511     my ($conf, $storecfg) = @_;
7512
7513     # first, return storage from conf if set
7514     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7515
7516     my ($target, $shared, $local);
7517
7518     foreach_storage_used_by_vm($conf, sub {
7519         my ($sid) = @_;
7520         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7521         my $dst = $scfg->{shared} ? \$shared : \$local;
7522         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7523     });
7524
7525     # second, use shared storage where VM has at least one disk
7526     # third, use local storage where VM has at least one disk
7527     # fall back to local storage
7528     $target = $shared // $local // 'local';
7529
7530     return $target;
7531 }
7532
7533 sub generate_uuid {
7534     my ($uuid, $uuid_str);
7535     UUID::generate($uuid);
7536     UUID::unparse($uuid, $uuid_str);
7537     return $uuid_str;
7538 }
7539
7540 sub generate_smbios1_uuid {
7541     return "uuid=".generate_uuid();
7542 }
7543
7544 sub nbd_stop {
7545     my ($vmid) = @_;
7546
7547     mon_cmd($vmid, 'nbd-server-stop');
7548 }
7549
7550 sub create_reboot_request {
7551     my ($vmid) = @_;
7552     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7553         or die "failed to create reboot trigger file: $!\n";
7554     close($fh);
7555 }
7556
7557 sub clear_reboot_request {
7558     my ($vmid) = @_;
7559     my $path = "/run/qemu-server/$vmid.reboot";
7560     my $res = 0;
7561
7562     $res = unlink($path);
7563     die "could not remove reboot request for $vmid: $!"
7564         if !$res && $! != POSIX::ENOENT;
7565
7566     return $res;
7567 }
7568
7569 sub bootorder_from_legacy {
7570     my ($conf, $bootcfg) = @_;
7571
7572     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
7573     my $bootindex_hash = {};
7574     my $i = 1;
7575     foreach my $o (split(//, $boot)) {
7576         $bootindex_hash->{$o} = $i*100;
7577         $i++;
7578     }
7579
7580     my $bootorder = {};
7581
7582     PVE::QemuConfig->foreach_volume($conf, sub {
7583         my ($ds, $drive) = @_;
7584
7585         if (drive_is_cdrom ($drive, 1)) {
7586             if ($bootindex_hash->{d}) {
7587                 $bootorder->{$ds} = $bootindex_hash->{d};
7588                 $bootindex_hash->{d} += 1;
7589             }
7590         } elsif ($bootindex_hash->{c}) {
7591             $bootorder->{$ds} = $bootindex_hash->{c}
7592                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
7593             $bootindex_hash->{c} += 1;
7594         }
7595     });
7596
7597     if ($bootindex_hash->{n}) {
7598         for (my $i = 0; $i < $MAX_NETS; $i++) {
7599             my $netname = "net$i";
7600             next if !$conf->{$netname};
7601             $bootorder->{$netname} = $bootindex_hash->{n};
7602             $bootindex_hash->{n} += 1;
7603         }
7604     }
7605
7606     return $bootorder;
7607 }
7608
7609 # Generate default device list for 'boot: order=' property. Matches legacy
7610 # default boot order, but with explicit device names. This is important, since
7611 # the fallback for when neither 'order' nor the old format is specified relies
7612 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
7613 sub get_default_bootdevices {
7614     my ($conf) = @_;
7615
7616     my @ret = ();
7617
7618     # harddisk
7619     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
7620     push @ret, $first if $first;
7621
7622     # cdrom
7623     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
7624     push @ret, $first if $first;
7625
7626     # network
7627     for (my $i = 0; $i < $MAX_NETS; $i++) {
7628         my $netname = "net$i";
7629         next if !$conf->{$netname};
7630         push @ret, $netname;
7631         last;
7632     }
7633
7634     return \@ret;
7635 }
7636
7637 sub device_bootorder {
7638     my ($conf) = @_;
7639
7640     return bootorder_from_legacy($conf) if !defined($conf->{boot});
7641
7642     my $boot = parse_property_string($boot_fmt, $conf->{boot});
7643
7644     my $bootorder = {};
7645     if (!defined($boot) || $boot->{legacy}) {
7646         $bootorder = bootorder_from_legacy($conf, $boot);
7647     } elsif ($boot->{order}) {
7648         my $i = 100; # start at 100 to allow user to insert devices before us with -args
7649         for my $dev (PVE::Tools::split_list($boot->{order})) {
7650             $bootorder->{$dev} = $i++;
7651         }
7652     }
7653
7654     return $bootorder;
7655 }
7656
7657 sub register_qmeventd_handle {
7658     my ($vmid) = @_;
7659
7660     my $fh;
7661     my $peer = "/var/run/qmeventd.sock";
7662     my $count = 0;
7663
7664     for (;;) {
7665         $count++;
7666         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
7667         last if $fh;
7668         if ($! != EINTR && $! != EAGAIN) {
7669             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
7670         }
7671         if ($count > 4) {
7672             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
7673               . "after $count retries\n";
7674         }
7675         usleep(25000);
7676     }
7677
7678     # send handshake to mark VM as backing up
7679     print $fh to_json({vzdump => {vmid => "$vmid"}});
7680
7681     # return handle to be closed later when inhibit is no longer required
7682     return $fh;
7683 }
7684
7685 # bash completion helper
7686
7687 sub complete_backup_archives {
7688     my ($cmdname, $pname, $cvalue) = @_;
7689
7690     my $cfg = PVE::Storage::config();
7691
7692     my $storeid;
7693
7694     if ($cvalue =~ m/^([^:]+):/) {
7695         $storeid = $1;
7696     }
7697
7698     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7699
7700     my $res = [];
7701     foreach my $id (keys %$data) {
7702         foreach my $item (@{$data->{$id}}) {
7703             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
7704             push @$res, $item->{volid} if defined($item->{volid});
7705         }
7706     }
7707
7708     return $res;
7709 }
7710
7711 my $complete_vmid_full = sub {
7712     my ($running) = @_;
7713
7714     my $idlist = vmstatus();
7715
7716     my $res = [];
7717
7718     foreach my $id (keys %$idlist) {
7719         my $d = $idlist->{$id};
7720         if (defined($running)) {
7721             next if $d->{template};
7722             next if $running && $d->{status} ne 'running';
7723             next if !$running && $d->{status} eq 'running';
7724         }
7725         push @$res, $id;
7726
7727     }
7728     return $res;
7729 };
7730
7731 sub complete_vmid {
7732     return &$complete_vmid_full();
7733 }
7734
7735 sub complete_vmid_stopped {
7736     return &$complete_vmid_full(0);
7737 }
7738
7739 sub complete_vmid_running {
7740     return &$complete_vmid_full(1);
7741 }
7742
7743 sub complete_storage {
7744
7745     my $cfg = PVE::Storage::config();
7746     my $ids = $cfg->{ids};
7747
7748     my $res = [];
7749     foreach my $sid (keys %$ids) {
7750         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7751         next if !$ids->{$sid}->{content}->{images};
7752         push @$res, $sid;
7753     }
7754
7755     return $res;
7756 }
7757
7758 sub complete_migration_storage {
7759     my ($cmd, $param, $current_value, $all_args) = @_;
7760
7761     my $targetnode = @$all_args[1];
7762
7763     my $cfg = PVE::Storage::config();
7764     my $ids = $cfg->{ids};
7765
7766     my $res = [];
7767     foreach my $sid (keys %$ids) {
7768         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7769         next if !$ids->{$sid}->{content}->{images};
7770         push @$res, $sid;
7771     }
7772
7773     return $res;
7774 }
7775
7776 sub vm_is_paused {
7777     my ($vmid) = @_;
7778     my $qmpstatus = eval {
7779         PVE::QemuConfig::assert_config_exists_on_node($vmid);
7780         mon_cmd($vmid, "query-status");
7781     };
7782     warn "$@\n" if $@;
7783     return $qmpstatus && $qmpstatus->{status} eq "paused";
7784 }
7785
7786 sub check_volume_storage_type {
7787     my ($storecfg, $vol) = @_;
7788
7789     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
7790     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7791     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
7792
7793     die "storage '$storeid' does not support content-type '$vtype'\n"
7794         if !$scfg->{content}->{$vtype};
7795
7796     return 1;
7797 }
7798
7799 1;