PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::CGroup;
  31 use PVE::DataCenterConfig;
  32 use PVE::Exception qw(raise raise_param_exc);
  33 use PVE::Format qw(render_duration render_bytes);
  34 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  35 use PVE::INotify;
  36 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  37 use PVE::ProcFSTools;
  38 use PVE::PBSClient;
  39 use PVE::RPCEnvironment;
  40 use PVE::Storage;
  41 use PVE::SysFSTools;
  42 use PVE::Systemd;
  43 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  44
  45 use PVE::QMPClient;
  46 use PVE::QemuConfig;
  47 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  48 use PVE::QemuServer::Cloudinit;
  49 use PVE::QemuServer::CGroup;
  50 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  51 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  52 use PVE::QemuServer::Machine;
  53 use PVE::QemuServer::Memory;
  54 use PVE::QemuServer::Monitor qw(mon_cmd);
  55 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  56 use PVE::QemuServer::USB qw(parse_usb_device);
  57
  58 my $have_sdn;
  59 eval {
  60     require PVE::Network::SDN::Zones;
  61     $have_sdn = 1;
  62 };
  63
  64 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  65 my $OVMF = {
  66     x86_64 => [
  67         "$EDK2_FW_BASE/OVMF_CODE.fd",
  68         "$EDK2_FW_BASE/OVMF_VARS.fd"
  69     ],
  70     aarch64 => [
  71         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  72         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  73     ],
  74 };
  75
  76 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  77
  78 # Note about locking: we use flock on the config file protect
  79 # against concurent actions.
  80 # Aditionaly, we have a 'lock' setting in the config file. This
  81 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  82 # allowed when such lock is set. But you can ignore this kind of
  83 # lock with the --skiplock flag.
  84
  85 cfs_register_file('/qemu-server/',
  86                   \&parse_vm_config,
  87                   \&write_vm_config);
  88
  89 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  90     description => "Some command save/restore state from this location.",
  91     type => 'string',
  92     maxLength => 128,
  93     optional => 1,
  94 });
  95
  96 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  97         description => "Specifies the Qemu machine type.",
  98         type => 'string',
  99         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 100         maxLength => 40,
 101         optional => 1,
 102 });
 103
 104
 105 sub map_storage {
 106     my ($map, $source) = @_;
 107
 108     return $source if !defined($map);
 109
 110     return $map->{entries}->{$source}
 111         if $map->{entries} && defined($map->{entries}->{$source});
 112
 113     return $map->{default} if $map->{default};
 114
 115     # identity (fallback)
 116     return $source;
 117 }
 118
 119 PVE::JSONSchema::register_standard_option('pve-targetstorage', {
 120     description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
 121     type => 'string',
 122     format => 'storagepair-list',
 123     optional => 1,
 124 });
 125
 126 #no warnings 'redefine';
 127
 128 my $nodename_cache;
 129 sub nodename {
 130     $nodename_cache //= PVE::INotify::nodename();
 131     return $nodename_cache;
 132 }
 133
 134 my $watchdog_fmt = {
 135     model => {
 136         default_key => 1,
 137         type => 'string',
 138         enum => [qw(i6300esb ib700)],
 139         description => "Watchdog type to emulate.",
 140         default => 'i6300esb',
 141         optional => 1,
 142     },
 143     action => {
 144         type => 'string',
 145         enum => [qw(reset shutdown poweroff pause debug none)],
 146         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 147         optional => 1,
 148     },
 149 };
 150 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 151
 152 my $agent_fmt = {
 153     enabled => {
 154         description => "Enable/disable Qemu GuestAgent.",
 155         type => 'boolean',
 156         default => 0,
 157         default_key => 1,
 158     },
 159     fstrim_cloned_disks => {
 160         description => "Run fstrim after moving a disk or migrating the VM.",
 161         type => 'boolean',
 162         optional => 1,
 163         default => 0
 164     },
 165     type => {
 166         description => "Select the agent type",
 167         type => 'string',
 168         default => 'virtio',
 169         optional => 1,
 170         enum => [qw(virtio isa)],
 171     },
 172 };
 173
 174 my $vga_fmt = {
 175     type => {
 176         description => "Select the VGA type.",
 177         type => 'string',
 178         default => 'std',
 179         optional => 1,
 180         default_key => 1,
 181         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 182     },
 183     memory => {
 184         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 185         type => 'integer',
 186         optional => 1,
 187         minimum => 4,
 188         maximum => 512,
 189     },
 190 };
 191
 192 my $ivshmem_fmt = {
 193     size => {
 194         type => 'integer',
 195         minimum => 1,
 196         description => "The size of the file in MB.",
 197     },
 198     name => {
 199         type => 'string',
 200         pattern => '[a-zA-Z0-9\-]+',
 201         optional => 1,
 202         format_description => 'string',
 203         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 204     },
 205 };
 206
 207 my $audio_fmt = {
 208     device => {
 209         type => 'string',
 210         enum => [qw(ich9-intel-hda intel-hda AC97)],
 211         description =>  "Configure an audio device."
 212     },
 213     driver =>  {
 214         type => 'string',
 215         enum => ['spice', 'none'],
 216         default => 'spice',
 217         optional => 1,
 218         description => "Driver backend for the audio device."
 219     },
 220 };
 221
 222 my $spice_enhancements_fmt = {
 223     foldersharing => {
 224         type => 'boolean',
 225         optional => 1,
 226         default => '0',
 227         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 228     },
 229     videostreaming =>  {
 230         type => 'string',
 231         enum => ['off', 'all', 'filter'],
 232         default => 'off',
 233         optional => 1,
 234         description => "Enable video streaming. Uses compression for detected video streams."
 235     },
 236 };
 237
 238 my $rng_fmt = {
 239     source => {
 240         type => 'string',
 241         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 242         default_key => 1,
 243         description => "The file on the host to gather entropy from. In most"
 244                      . " cases /dev/urandom should be preferred over /dev/random"
 245                      . " to avoid entropy-starvation issues on the host. Using"
 246                      . " urandom does *not* decrease security in any meaningful"
 247                      . " way, as it's still seeded from real entropy, and the"
 248                      . " bytes provided will most likely be mixed with real"
 249                      . " entropy on the guest as well. /dev/hwrng can be used"
 250                      . " to pass through a hardware RNG from the host.",
 251     },
 252     max_bytes => {
 253         type => 'integer',
 254         description => "Maximum bytes of entropy injected into the guest every"
 255                      . " 'period' milliseconds. Prefer a lower value when using"
 256                      . " /dev/random as source. Use 0 to disable limiting"
 257                      . " (potentially dangerous!).",
 258         optional => 1,
 259
 260         # default is 1 KiB/s, provides enough entropy to the guest to avoid
 261         # boot-starvation issues (e.g. systemd etc...) while allowing no chance
 262         # of overwhelming the host, provided we're reading from /dev/urandom
 263         default => 1024,
 264     },
 265     period => {
 266         type => 'integer',
 267         description => "Every 'period' milliseconds the entropy-injection quota"
 268                      . " is reset, allowing the guest to retrieve another"
 269                      . " 'max_bytes' of entropy.",
 270         optional => 1,
 271         default => 1000,
 272     },
 273 };
 274
 275 my $confdesc = {
 276     onboot => {
 277         optional => 1,
 278         type => 'boolean',
 279         description => "Specifies whether a VM will be started during system bootup.",
 280         default => 0,
 281     },
 282     autostart => {
 283         optional => 1,
 284         type => 'boolean',
 285         description => "Automatic restart after crash (currently ignored).",
 286         default => 0,
 287     },
 288     hotplug => {
 289         optional => 1,
 290         type => 'string', format => 'pve-hotplug-features',
 291         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 292         default => 'network,disk,usb',
 293     },
 294     reboot => {
 295         optional => 1,
 296         type => 'boolean',
 297         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 298         default => 1,
 299     },
 300     lock => {
 301         optional => 1,
 302         type => 'string',
 303         description => "Lock/unlock the VM.",
 304         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 305     },
 306     cpulimit => {
 307         optional => 1,
 308         type => 'number',
 309         description => "Limit of CPU usage.",
 310         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 311         minimum => 0,
 312         maximum => 128,
 313         default => 0,
 314     },
 315     cpuunits => {
 316         optional => 1,
 317         type => 'integer',
 318         description => "CPU weight for a VM.",
 319         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 320         minimum => 2,
 321         maximum => 262144,
 322         default => 1024,
 323     },
 324     memory => {
 325         optional => 1,
 326         type => 'integer',
 327         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 328         minimum => 16,
 329         default => 512,
 330     },
 331     balloon => {
 332         optional => 1,
 333         type => 'integer',
 334         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 335         minimum => 0,
 336     },
 337     shares => {
 338         optional => 1,
 339         type => 'integer',
 340         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 341         minimum => 0,
 342         maximum => 50000,
 343         default => 1000,
 344     },
 345     keyboard => {
 346         optional => 1,
 347         type => 'string',
 348         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 349                        "It should not be necessary to set it.",
 350         enum => PVE::Tools::kvmkeymaplist(),
 351         default => undef,
 352     },
 353     name => {
 354         optional => 1,
 355         type => 'string', format => 'dns-name',
 356         description => "Set a name for the VM. Only used on the configuration web interface.",
 357     },
 358     scsihw => {
 359         optional => 1,
 360         type => 'string',
 361         description => "SCSI controller model",
 362         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 363         default => 'lsi',
 364     },
 365     description => {
 366         optional => 1,
 367         type => 'string',
 368         description => "Description for the VM. Shown in the web-interface VM's summary."
 369             ." This is saved as comment inside the configuration file.",
 370         maxLength => 1024 * 8,
 371     },
 372     ostype => {
 373         optional => 1,
 374         type => 'string',
 375         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 376         description => "Specify guest operating system.",
 377         verbose_description => <<EODESC,
 378 Specify guest operating system. This is used to enable special
 379 optimization/features for specific operating systems:
 380
 381 [horizontal]
 382 other;; unspecified OS
 383 wxp;; Microsoft Windows XP
 384 w2k;; Microsoft Windows 2000
 385 w2k3;; Microsoft Windows 2003
 386 w2k8;; Microsoft Windows 2008
 387 wvista;; Microsoft Windows Vista
 388 win7;; Microsoft Windows 7
 389 win8;; Microsoft Windows 8/2012/2012r2
 390 win10;; Microsoft Windows 10/2016/2019
 391 l24;; Linux 2.4 Kernel
 392 l26;; Linux 2.6 - 5.X Kernel
 393 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 394 EODESC
 395     },
 396     boot => {
 397         optional => 1,
 398         type => 'string', format => 'pve-qm-boot',
 399         description => "Specify guest boot order. Use with 'order=', usage with"
 400                      . " no key or 'legacy=' is deprecated.",
 401     },
 402     bootdisk => {
 403         optional => 1,
 404         type => 'string', format => 'pve-qm-bootdisk',
 405         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 406         pattern => '(ide|sata|scsi|virtio)\d+',
 407     },
 408     smp => {
 409         optional => 1,
 410         type => 'integer',
 411         description => "The number of CPUs. Please use option -sockets instead.",
 412         minimum => 1,
 413         default => 1,
 414     },
 415     sockets => {
 416         optional => 1,
 417         type => 'integer',
 418         description => "The number of CPU sockets.",
 419         minimum => 1,
 420         default => 1,
 421     },
 422     cores => {
 423         optional => 1,
 424         type => 'integer',
 425         description => "The number of cores per socket.",
 426         minimum => 1,
 427         default => 1,
 428     },
 429     numa => {
 430         optional => 1,
 431         type => 'boolean',
 432         description => "Enable/disable NUMA.",
 433         default => 0,
 434     },
 435     hugepages => {
 436         optional => 1,
 437         type => 'string',
 438         description => "Enable/disable hugepages memory.",
 439         enum => [qw(any 2 1024)],
 440     },
 441     keephugepages => {
 442         optional => 1,
 443         type => 'boolean',
 444         default => 0,
 445         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 446             ." after VM shutdown and can be used for subsequent starts.",
 447     },
 448     vcpus => {
 449         optional => 1,
 450         type => 'integer',
 451         description => "Number of hotplugged vcpus.",
 452         minimum => 1,
 453         default => 0,
 454     },
 455     acpi => {
 456         optional => 1,
 457         type => 'boolean',
 458         description => "Enable/disable ACPI.",
 459         default => 1,
 460     },
 461     agent => {
 462         optional => 1,
 463         description => "Enable/disable Qemu GuestAgent and its properties.",
 464         type => 'string',
 465         format => $agent_fmt,
 466     },
 467     kvm => {
 468         optional => 1,
 469         type => 'boolean',
 470         description => "Enable/disable KVM hardware virtualization.",
 471         default => 1,
 472     },
 473     tdf => {
 474         optional => 1,
 475         type => 'boolean',
 476         description => "Enable/disable time drift fix.",
 477         default => 0,
 478     },
 479     localtime => {
 480         optional => 1,
 481         type => 'boolean',
 482         description => "Set the real time clock to local time. This is enabled by default if ostype"
 483             ." indicates a Microsoft OS.",
 484     },
 485     freeze => {
 486         optional => 1,
 487         type => 'boolean',
 488         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 489     },
 490     vga => {
 491         optional => 1,
 492         type => 'string', format => $vga_fmt,
 493         description => "Configure the VGA hardware.",
 494         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 495             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 496             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 497             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 498             ." display server. For win* OS you can select how many independent displays you want,"
 499             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 500             ." using a serial device as terminal.",
 501     },
 502     watchdog => {
 503         optional => 1,
 504         type => 'string', format => 'pve-qm-watchdog',
 505         description => "Create a virtual hardware watchdog device.",
 506         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 507             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 508             ." else the watchdog will reset the guest (or execute the respective action specified)",
 509     },
 510     startdate => {
 511         optional => 1,
 512         type => 'string',
 513         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 514         description => "Set the initial date of the real time clock. Valid format for date are:"
 515             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 516         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 517         default => 'now',
 518     },
 519     startup =>  get_standard_option('pve-startup-order'),
 520     template => {
 521         optional => 1,
 522         type => 'boolean',
 523         description => "Enable/disable Template.",
 524         default => 0,
 525     },
 526     args => {
 527         optional => 1,
 528         type => 'string',
 529         description => "Arbitrary arguments passed to kvm.",
 530         verbose_description => <<EODESCR,
 531 Arbitrary arguments passed to kvm, for example:
 532
 533 args: -no-reboot -no-hpet
 534
 535 NOTE: this option is for experts only.
 536 EODESCR
 537     },
 538     tablet => {
 539         optional => 1,
 540         type => 'boolean',
 541         default => 1,
 542         description => "Enable/disable the USB tablet device.",
 543         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 544             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 545             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 546             ." may consider disabling this to save some context switches. This is turned off by"
 547             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 548     },
 549     migrate_speed => {
 550         optional => 1,
 551         type => 'integer',
 552         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 553         minimum => 0,
 554         default => 0,
 555     },
 556     migrate_downtime => {
 557         optional => 1,
 558         type => 'number',
 559         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 560         minimum => 0,
 561         default => 0.1,
 562     },
 563     cdrom => {
 564         optional => 1,
 565         type => 'string', format => 'pve-qm-ide',
 566         typetext => '<volume>',
 567         description => "This is an alias for option -ide2",
 568     },
 569     cpu => {
 570         optional => 1,
 571         description => "Emulated CPU type.",
 572         type => 'string',
 573         format => 'pve-vm-cpu-conf',
 574     },
 575     parent => get_standard_option('pve-snapshot-name', {
 576         optional => 1,
 577         description => "Parent snapshot name. This is used internally, and should not be modified.",
 578     }),
 579     snaptime => {
 580         optional => 1,
 581         description => "Timestamp for snapshots.",
 582         type => 'integer',
 583         minimum => 0,
 584     },
 585     vmstate => {
 586         optional => 1,
 587         type => 'string', format => 'pve-volume-id',
 588         description => "Reference to a volume which stores the VM state. This is used internally"
 589             ." for snapshots.",
 590     },
 591     vmstatestorage => get_standard_option('pve-storage-id', {
 592         description => "Default storage for VM state volumes/files.",
 593         optional => 1,
 594     }),
 595     runningmachine => get_standard_option('pve-qemu-machine', {
 596         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 597             ." for snapshots.",
 598     }),
 599     runningcpu => {
 600         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 601             ." internally for snapshots.",
 602         optional => 1,
 603         type => 'string',
 604         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 605         format_description => 'QEMU -cpu parameter'
 606     },
 607     machine => get_standard_option('pve-qemu-machine'),
 608     arch => {
 609         description => "Virtual processor architecture. Defaults to the host.",
 610         optional => 1,
 611         type => 'string',
 612         enum => [qw(x86_64 aarch64)],
 613     },
 614     smbios1 => {
 615         description => "Specify SMBIOS type 1 fields.",
 616         type => 'string', format => 'pve-qm-smbios1',
 617         maxLength => 512,
 618         optional => 1,
 619     },
 620     protection => {
 621         optional => 1,
 622         type => 'boolean',
 623         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 624             ." remove disk operations.",
 625         default => 0,
 626     },
 627     bios => {
 628         optional => 1,
 629         type => 'string',
 630         enum => [ qw(seabios ovmf) ],
 631         description => "Select BIOS implementation.",
 632         default => 'seabios',
 633     },
 634     vmgenid => {
 635         type => 'string',
 636         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 637         format_description => 'UUID',
 638         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 639             ." to disable explicitly.",
 640         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 641             ." value identifier to the guest OS. This allows to notify the guest operating system"
 642             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 643             ." execution or creation from a template). The guest operating system notices the"
 644             ." change, and is then able to react as appropriate by marking its copies of"
 645             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 646             ."Note that auto-creation only works when done through API/CLI create or update methods"
 647             .", but not when manually editing the config file.",
 648         default => "1 (autogenerated)",
 649         optional => 1,
 650     },
 651     hookscript => {
 652         type => 'string',
 653         format => 'pve-volume-id',
 654         optional => 1,
 655         description => "Script that will be executed during various steps in the vms lifetime.",
 656     },
 657     ivshmem => {
 658         type => 'string',
 659         format => $ivshmem_fmt,
 660         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 661             ." the host.",
 662         optional => 1,
 663     },
 664     audio0 => {
 665         type => 'string',
 666         format => $audio_fmt,
 667         description => "Configure a audio device, useful in combination with QXL/Spice.",
 668         optional => 1
 669     },
 670     spice_enhancements => {
 671         type => 'string',
 672         format => $spice_enhancements_fmt,
 673         description => "Configure additional enhancements for SPICE.",
 674         optional => 1
 675     },
 676     tags => {
 677         type => 'string', format => 'pve-tag-list',
 678         description => 'Tags of the VM. This is only meta information.',
 679         optional => 1,
 680     },
 681     rng0 => {
 682         type => 'string',
 683         format => $rng_fmt,
 684         description => "Configure a VirtIO-based Random Number Generator.",
 685         optional => 1,
 686     },
 687 };
 688
 689 my $cicustom_fmt = {
 690     meta => {
 691         type => 'string',
 692         optional => 1,
 693         description => 'Specify a custom file containing all meta data passed to the VM via"
 694             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 695         format => 'pve-volume-id',
 696         format_description => 'volume',
 697     },
 698     network => {
 699         type => 'string',
 700         optional => 1,
 701         description => 'Specify a custom file containing all network data passed to the VM via'
 702             .' cloud-init.',
 703         format => 'pve-volume-id',
 704         format_description => 'volume',
 705     },
 706     user => {
 707         type => 'string',
 708         optional => 1,
 709         description => 'Specify a custom file containing all user data passed to the VM via'
 710             .' cloud-init.',
 711         format => 'pve-volume-id',
 712         format_description => 'volume',
 713     },
 714 };
 715 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 716
 717 my $confdesc_cloudinit = {
 718     citype => {
 719         optional => 1,
 720         type => 'string',
 721         description => 'Specifies the cloud-init configuration format. The default depends on the'
 722             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 723             .' and `configdrive2` for windows.',
 724         enum => ['configdrive2', 'nocloud', 'opennebula'],
 725     },
 726     ciuser => {
 727         optional => 1,
 728         type => 'string',
 729         description => "cloud-init: User name to change ssh keys and password for instead of the"
 730             ." image's configured default user.",
 731     },
 732     cipassword => {
 733         optional => 1,
 734         type => 'string',
 735         description => 'cloud-init: Password to assign the user. Using this is generally not'
 736             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 737             .' support hashed passwords.',
 738     },
 739     cicustom => {
 740         optional => 1,
 741         type => 'string',
 742         description => 'cloud-init: Specify custom files to replace the automatically generated'
 743             .' ones at start.',
 744         format => 'pve-qm-cicustom',
 745     },
 746     searchdomain => {
 747         optional => 1,
 748         type => 'string',
 749         description => "cloud-init: Sets DNS search domains for a container. Create will'
 750             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 751             .' are set.",
 752     },
 753     nameserver => {
 754         optional => 1,
 755         type => 'string', format => 'address-list',
 756         description => "cloud-init: Sets DNS server IP address for a container. Create will'
 757             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 758             .' are set.",
 759     },
 760     sshkeys => {
 761         optional => 1,
 762         type => 'string',
 763         format => 'urlencoded',
 764         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 765     },
 766 };
 767
 768 # what about other qemu settings ?
 769 #cpu => 'string',
 770 #machine => 'string',
 771 #fda => 'file',
 772 #fdb => 'file',
 773 #mtdblock => 'file',
 774 #sd => 'file',
 775 #pflash => 'file',
 776 #snapshot => 'bool',
 777 #bootp => 'file',
 778 ##tftp => 'dir',
 779 ##smb => 'dir',
 780 #kernel => 'file',
 781 #append => 'string',
 782 #initrd => 'file',
 783 ##soundhw => 'string',
 784
 785 while (my ($k, $v) = each %$confdesc) {
 786     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 787 }
 788
 789 my $MAX_USB_DEVICES = 5;
 790 my $MAX_NETS = 32;
 791 my $MAX_SERIAL_PORTS = 4;
 792 my $MAX_PARALLEL_PORTS = 3;
 793 my $MAX_NUMA = 8;
 794
 795 my $numa_fmt = {
 796     cpus => {
 797         type => "string",
 798         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 799         description => "CPUs accessing this NUMA node.",
 800         format_description => "id[-id];...",
 801     },
 802     memory => {
 803         type => "number",
 804         description => "Amount of memory this NUMA node provides.",
 805         optional => 1,
 806     },
 807     hostnodes => {
 808         type => "string",
 809         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 810         description => "Host NUMA nodes to use.",
 811         format_description => "id[-id];...",
 812         optional => 1,
 813     },
 814     policy => {
 815         type => 'string',
 816         enum => [qw(preferred bind interleave)],
 817         description => "NUMA allocation policy.",
 818         optional => 1,
 819     },
 820 };
 821 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 822 my $numadesc = {
 823     optional => 1,
 824     type => 'string', format => $numa_fmt,
 825     description => "NUMA topology.",
 826 };
 827 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 828
 829 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 830     $confdesc->{"numa$i"} = $numadesc;
 831 }
 832
 833 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 834                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 835                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 836 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 837
 838 my $net_fmt_bridge_descr = <<__EOD__;
 839 Bridge to attach the network device to. The Proxmox VE standard bridge
 840 is called 'vmbr0'.
 841
 842 If you do not specify a bridge, we create a kvm user (NATed) network
 843 device, which provides DHCP and DNS services. The following addresses
 844 are used:
 845
 846  10.0.2.2   Gateway
 847  10.0.2.3   DNS Server
 848  10.0.2.4   SMB Server
 849
 850 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 851 __EOD__
 852
 853 my $net_fmt = {
 854     macaddr  => get_standard_option('mac-addr', {
 855         description => "MAC address. That address must be unique withing your network. This is"
 856             ." automatically generated if not specified.",
 857     }),
 858     model => {
 859         type => 'string',
 860         description => "Network Card Model. The 'virtio' model provides the best performance with"
 861             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 862             ." best to use 'e1000'.",
 863         enum => $nic_model_list,
 864         default_key => 1,
 865     },
 866     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 867     bridge => {
 868         type => 'string',
 869         description => $net_fmt_bridge_descr,
 870         format_description => 'bridge',
 871         pattern => '[-_.\w\d]+',
 872         optional => 1,
 873     },
 874     queues => {
 875         type => 'integer',
 876         minimum => 0, maximum => 16,
 877         description => 'Number of packet queues to be used on the device.',
 878         optional => 1,
 879     },
 880     rate => {
 881         type => 'number',
 882         minimum => 0,
 883         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 884         optional => 1,
 885     },
 886     tag => {
 887         type => 'integer',
 888         minimum => 1, maximum => 4094,
 889         description => 'VLAN tag to apply to packets on this interface.',
 890         optional => 1,
 891     },
 892     trunks => {
 893         type => 'string',
 894         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 895         description => 'VLAN trunks to pass through this interface.',
 896         format_description => 'vlanid[;vlanid...]',
 897         optional => 1,
 898     },
 899     firewall => {
 900         type => 'boolean',
 901         description => 'Whether this interface should be protected by the firewall.',
 902         optional => 1,
 903     },
 904     link_down => {
 905         type => 'boolean',
 906         description => 'Whether this interface should be disconnected (like pulling the plug).',
 907         optional => 1,
 908     },
 909     mtu => {
 910         type => 'integer',
 911         minimum => 1, maximum => 65520,
 912         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 913         optional => 1,
 914     },
 915 };
 916
 917 my $netdesc = {
 918     optional => 1,
 919     type => 'string', format => $net_fmt,
 920     description => "Specify network devices.",
 921 };
 922
 923 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 924
 925 my $ipconfig_fmt = {
 926     ip => {
 927         type => 'string',
 928         format => 'pve-ipv4-config',
 929         format_description => 'IPv4Format/CIDR',
 930         description => 'IPv4 address in CIDR format.',
 931         optional => 1,
 932         default => 'dhcp',
 933     },
 934     gw => {
 935         type => 'string',
 936         format => 'ipv4',
 937         format_description => 'GatewayIPv4',
 938         description => 'Default gateway for IPv4 traffic.',
 939         optional => 1,
 940         requires => 'ip',
 941     },
 942     ip6 => {
 943         type => 'string',
 944         format => 'pve-ipv6-config',
 945         format_description => 'IPv6Format/CIDR',
 946         description => 'IPv6 address in CIDR format.',
 947         optional => 1,
 948         default => 'dhcp',
 949     },
 950     gw6 => {
 951         type => 'string',
 952         format => 'ipv6',
 953         format_description => 'GatewayIPv6',
 954         description => 'Default gateway for IPv6 traffic.',
 955         optional => 1,
 956         requires => 'ip6',
 957     },
 958 };
 959 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 960 my $ipconfigdesc = {
 961     optional => 1,
 962     type => 'string', format => 'pve-qm-ipconfig',
 963     description => <<'EODESCR',
 964 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 965
 966 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 967
 968 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
 969 gateway should be provided.
 970 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
 971 cloud-init 19.4 or newer.
 972
 973 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
 974 dhcp on IPv4.
 975 EODESCR
 976 };
 977 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 978
 979 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 980     $confdesc->{"net$i"} = $netdesc;
 981     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 982 }
 983
 984 foreach my $key (keys %$confdesc_cloudinit) {
 985     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 986 }
 987
 988 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 989 sub verify_volume_id_or_qm_path {
 990     my ($volid, $noerr) = @_;
 991
 992     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 993         return $volid;
 994     }
 995
 996     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 997     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 998     if ($@) {
 999         return if $noerr;
1000         die $@;
1001     }
1002     return $volid;
1003 }
1004
1005 my $usb_fmt = {
1006     host => {
1007         default_key => 1,
1008         type => 'string', format => 'pve-qm-usb-device',
1009         format_description => 'HOSTUSBDEVICE|spice',
1010         description => <<EODESCR,
1011 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1012
1013  'bus-port(.port)*' (decimal numbers) or
1014  'vendor_id:product_id' (hexadeciaml numbers) or
1015  'spice'
1016
1017 You can use the 'lsusb -t' command to list existing usb devices.
1018
1019 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1020 machines - use with special care.
1021
1022 The value 'spice' can be used to add a usb redirection devices for spice.
1023 EODESCR
1024     },
1025     usb3 => {
1026         optional => 1,
1027         type => 'boolean',
1028         description => "Specifies whether if given host option is a USB3 device or port.",
1029         default => 0,
1030     },
1031 };
1032
1033 my $usbdesc = {
1034     optional => 1,
1035     type => 'string', format => $usb_fmt,
1036     description => "Configure an USB device (n is 0 to 4).",
1037 };
1038 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1039
1040 my $serialdesc = {
1041         optional => 1,
1042         type => 'string',
1043         pattern => '(/dev/.+|socket)',
1044         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1045         verbose_description =>  <<EODESCR,
1046 Create a serial device inside the VM (n is 0 to 3), and pass through a
1047 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1048 host side (use 'qm terminal' to open a terminal connection).
1049
1050 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1051 use with special care.
1052
1053 CAUTION: Experimental! User reported problems with this option.
1054 EODESCR
1055 };
1056
1057 my $paralleldesc= {
1058         optional => 1,
1059         type => 'string',
1060         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1061         description =>  "Map host parallel devices (n is 0 to 2).",
1062         verbose_description =>  <<EODESCR,
1063 Map host parallel devices (n is 0 to 2).
1064
1065 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1066 machines - use with special care.
1067
1068 CAUTION: Experimental! User reported problems with this option.
1069 EODESCR
1070 };
1071
1072 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1073     $confdesc->{"parallel$i"} = $paralleldesc;
1074 }
1075
1076 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1077     $confdesc->{"serial$i"} = $serialdesc;
1078 }
1079
1080 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1081     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1082 }
1083
1084 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1085     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1086 }
1087
1088 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1089     $confdesc->{"usb$i"} = $usbdesc;
1090 }
1091
1092 my $boot_fmt = {
1093     legacy => {
1094         optional => 1,
1095         default_key => 1,
1096         type => 'string',
1097         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1098                      . " Deprecated, use 'order=' instead.",
1099         pattern => '[acdn]{1,4}',
1100         format_description => "[acdn]{1,4}",
1101
1102         # note: this is also the fallback if boot: is not given at all
1103         default => 'cdn',
1104     },
1105     order => {
1106         optional => 1,
1107         type => 'string',
1108         format => 'pve-qm-bootdev-list',
1109         format_description => "device[;device...]",
1110         description => <<EODESC,
1111 The guest will attempt to boot from devices in the order they appear here.
1112
1113 Disks, optical drives and passed-through storage USB devices will be directly
1114 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1115 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1116
1117 Note that only devices in this list will be marked as bootable and thus loaded
1118 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1119 (e.g. software-raid), you need to specify all of them here.
1120
1121 Overrides the deprecated 'legacy=[acdn]*' value when given.
1122 EODESC
1123     },
1124 };
1125 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1126
1127 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1128 sub verify_bootdev {
1129     my ($dev, $noerr) = @_;
1130
1131     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && $dev !~ m/^efidisk/;
1132
1133     my $check = sub {
1134         my ($base) = @_;
1135         return 0 if $dev !~ m/^$base\d+$/;
1136         return 0 if !$confdesc->{$dev};
1137         return 1;
1138     };
1139
1140     return $dev if $check->("net");
1141     return $dev if $check->("usb");
1142     return $dev if $check->("hostpci");
1143
1144     return if $noerr;
1145     die "invalid boot device '$dev'\n";
1146 }
1147
1148 sub print_bootorder {
1149     my ($devs) = @_;
1150     return "" if !@$devs;
1151     my $data = { order => join(';', @$devs) };
1152     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1153 }
1154
1155 my $kvm_api_version = 0;
1156
1157 sub kvm_version {
1158     return $kvm_api_version if $kvm_api_version;
1159
1160     open my $fh, '<', '/dev/kvm' or return;
1161
1162     # 0xae00 => KVM_GET_API_VERSION
1163     $kvm_api_version = ioctl($fh, 0xae00, 0);
1164     close($fh);
1165
1166     return $kvm_api_version;
1167 }
1168
1169 my $kvm_user_version = {};
1170 my $kvm_mtime = {};
1171
1172 sub kvm_user_version {
1173     my ($binary) = @_;
1174
1175     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1176     my $st = stat($binary);
1177
1178     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1179     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1180         $cachedmtime == $st->mtime;
1181
1182     $kvm_user_version->{$binary} = 'unknown';
1183     $kvm_mtime->{$binary} = $st->mtime;
1184
1185     my $code = sub {
1186         my $line = shift;
1187         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1188             $kvm_user_version->{$binary} = $2;
1189         }
1190     };
1191
1192     eval { run_command([$binary, '--version'], outfunc => $code); };
1193     warn $@ if $@;
1194
1195     return $kvm_user_version->{$binary};
1196
1197 }
1198 my sub extract_version {
1199     my ($machine_type, $version) = @_;
1200     $version = kvm_user_version() if !defined($version);
1201     PVE::QemuServer::Machine::extract_version($machine_type, $version)
1202 }
1203
1204 sub kernel_has_vhost_net {
1205     return -c '/dev/vhost-net';
1206 }
1207
1208 sub option_exists {
1209     my $key = shift;
1210     return defined($confdesc->{$key});
1211 }
1212
1213 my $cdrom_path;
1214 sub get_cdrom_path {
1215
1216     return  $cdrom_path if $cdrom_path;
1217
1218     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1219     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1220     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1221 }
1222
1223 sub get_iso_path {
1224     my ($storecfg, $vmid, $cdrom) = @_;
1225
1226     if ($cdrom eq 'cdrom') {
1227         return get_cdrom_path();
1228     } elsif ($cdrom eq 'none') {
1229         return '';
1230     } elsif ($cdrom =~ m|^/|) {
1231         return $cdrom;
1232     } else {
1233         return PVE::Storage::path($storecfg, $cdrom);
1234     }
1235 }
1236
1237 # try to convert old style file names to volume IDs
1238 sub filename_to_volume_id {
1239     my ($vmid, $file, $media) = @_;
1240
1241      if (!($file eq 'none' || $file eq 'cdrom' ||
1242           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1243
1244         return if $file =~ m|/|;
1245
1246         if ($media && $media eq 'cdrom') {
1247             $file = "local:iso/$file";
1248         } else {
1249             $file = "local:$vmid/$file";
1250         }
1251     }
1252
1253     return $file;
1254 }
1255
1256 sub verify_media_type {
1257     my ($opt, $vtype, $media) = @_;
1258
1259     return if !$media;
1260
1261     my $etype;
1262     if ($media eq 'disk') {
1263         $etype = 'images';
1264     } elsif ($media eq 'cdrom') {
1265         $etype = 'iso';
1266     } else {
1267         die "internal error";
1268     }
1269
1270     return if ($vtype eq $etype);
1271
1272     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1273 }
1274
1275 sub cleanup_drive_path {
1276     my ($opt, $storecfg, $drive) = @_;
1277
1278     # try to convert filesystem paths to volume IDs
1279
1280     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1281         ($drive->{file} !~ m|^/dev/.+|) &&
1282         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1283         ($drive->{file} !~ m/^\d+$/)) {
1284         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1285         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1286             if !$vtype;
1287         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1288         verify_media_type($opt, $vtype, $drive->{media});
1289         $drive->{file} = $volid;
1290     }
1291
1292     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1293 }
1294
1295 sub parse_hotplug_features {
1296     my ($data) = @_;
1297
1298     my $res = {};
1299
1300     return $res if $data eq '0';
1301
1302     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1303
1304     foreach my $feature (PVE::Tools::split_list($data)) {
1305         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1306             $res->{$1} = 1;
1307         } else {
1308             die "invalid hotplug feature '$feature'\n";
1309         }
1310     }
1311     return $res;
1312 }
1313
1314 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1315 sub pve_verify_hotplug_features {
1316     my ($value, $noerr) = @_;
1317
1318     return $value if parse_hotplug_features($value);
1319
1320     return if $noerr;
1321
1322     die "unable to parse hotplug option\n";
1323 }
1324
1325 sub scsi_inquiry {
1326     my($fh, $noerr) = @_;
1327
1328     my $SG_IO = 0x2285;
1329     my $SG_GET_VERSION_NUM = 0x2282;
1330
1331     my $versionbuf = "\x00" x 8;
1332     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1333     if (!$ret) {
1334         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1335         return;
1336     }
1337     my $version = unpack("I", $versionbuf);
1338     if ($version < 30000) {
1339         die "scsi generic interface too old\n"  if !$noerr;
1340         return;
1341     }
1342
1343     my $buf = "\x00" x 36;
1344     my $sensebuf = "\x00" x 8;
1345     my $cmd = pack("C x3 C x1", 0x12, 36);
1346
1347     # see /usr/include/scsi/sg.h
1348     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1349
1350     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1351                       length($sensebuf), 0, length($buf), $buf,
1352                       $cmd, $sensebuf, 6000);
1353
1354     $ret = ioctl($fh, $SG_IO, $packet);
1355     if (!$ret) {
1356         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1357         return;
1358     }
1359
1360     my @res = unpack($sg_io_hdr_t, $packet);
1361     if ($res[17] || $res[18]) {
1362         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1363         return;
1364     }
1365
1366     my $res = {};
1367     (my $byte0, my $byte1, $res->{vendor},
1368      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1369
1370     $res->{removable} = $byte1 & 128 ? 1 : 0;
1371     $res->{type} = $byte0 & 31;
1372
1373     return $res;
1374 }
1375
1376 sub path_is_scsi {
1377     my ($path) = @_;
1378
1379     my $fh = IO::File->new("+<$path") || return;
1380     my $res = scsi_inquiry($fh, 1);
1381     close($fh);
1382
1383     return $res;
1384 }
1385
1386 sub print_tabletdevice_full {
1387     my ($conf, $arch) = @_;
1388
1389     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1390
1391     # we use uhci for old VMs because tablet driver was buggy in older qemu
1392     my $usbbus;
1393     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1394         $usbbus = 'ehci';
1395     } else {
1396         $usbbus = 'uhci';
1397     }
1398
1399     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1400 }
1401
1402 sub print_keyboarddevice_full {
1403     my ($conf, $arch, $machine) = @_;
1404
1405     return if $arch ne 'aarch64';
1406
1407     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1408 }
1409
1410 my sub get_drive_id {
1411     my ($drive) = @_;
1412     return "$drive->{interface}$drive->{index}";
1413 }
1414
1415 sub print_drivedevice_full {
1416     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1417
1418     my $device = '';
1419     my $maxdev = 0;
1420
1421     my $drive_id = get_drive_id($drive);
1422     if ($drive->{interface} eq 'virtio') {
1423         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1424         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1425         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1426     } elsif ($drive->{interface} eq 'scsi') {
1427
1428         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1429         my $unit = $drive->{index} % $maxdev;
1430         my $devicetype = 'hd';
1431         my $path = '';
1432         if (drive_is_cdrom($drive)) {
1433             $devicetype = 'cd';
1434         } else {
1435             if ($drive->{file} =~ m|^/|) {
1436                 $path = $drive->{file};
1437                 if (my $info = path_is_scsi($path)) {
1438                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1439                         $devicetype = 'block';
1440                     } elsif ($info->{type} == 1) { # tape
1441                         $devicetype = 'generic';
1442                     }
1443                 }
1444             } else {
1445                  $path = PVE::Storage::path($storecfg, $drive->{file});
1446             }
1447
1448             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1449             my $version = extract_version($machine_type, kvm_user_version());
1450             if ($path =~ m/^iscsi\:\/\// &&
1451                !min_version($version, 4, 1)) {
1452                 $devicetype = 'generic';
1453             }
1454         }
1455
1456         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1457             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1458         } else {
1459             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1460                 .",lun=$drive->{index}";
1461         }
1462         $device .= ",drive=drive-$drive_id,id=$drive_id";
1463
1464         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1465             $device .= ",rotation_rate=1";
1466         }
1467         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1468
1469     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1470         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1471         my $controller = int($drive->{index} / $maxdev);
1472         my $unit = $drive->{index} % $maxdev;
1473         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1474
1475         $device = "ide-$devicetype";
1476         if ($drive->{interface} eq 'ide') {
1477             $device .= ",bus=ide.$controller,unit=$unit";
1478         } else {
1479             $device .= ",bus=ahci$controller.$unit";
1480         }
1481         $device .= ",drive=drive-$drive_id,id=$drive_id";
1482
1483         if ($devicetype eq 'hd') {
1484             if (my $model = $drive->{model}) {
1485                 $model = URI::Escape::uri_unescape($model);
1486                 $device .= ",model=$model";
1487             }
1488             if ($drive->{ssd}) {
1489                 $device .= ",rotation_rate=1";
1490             }
1491         }
1492         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1493     } elsif ($drive->{interface} eq 'usb') {
1494         die "implement me";
1495         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1496     } else {
1497         die "unsupported interface type";
1498     }
1499
1500     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1501
1502     if (my $serial = $drive->{serial}) {
1503         $serial = URI::Escape::uri_unescape($serial);
1504         $device .= ",serial=$serial";
1505     }
1506
1507
1508     return $device;
1509 }
1510
1511 sub get_initiator_name {
1512     my $initiator;
1513
1514     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1515     while (defined(my $line = <$fh>)) {
1516         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1517         $initiator = $1;
1518         last;
1519     }
1520     $fh->close();
1521
1522     return $initiator;
1523 }
1524
1525 sub print_drive_commandline_full {
1526     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1527
1528     my $path;
1529     my $volid = $drive->{file};
1530     my $format = $drive->{format};
1531     my $drive_id = get_drive_id($drive);
1532
1533     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1534     my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1535
1536     if (drive_is_cdrom($drive)) {
1537         $path = get_iso_path($storecfg, $vmid, $volid);
1538         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1539     } else {
1540         if ($storeid) {
1541             $path = PVE::Storage::path($storecfg, $volid);
1542             $format //= qemu_img_format($scfg, $volname);
1543         } else {
1544             $path = $volid;
1545             $format //= "raw";
1546         }
1547    }
1548
1549    my $is_rbd = $path =~ m/^rbd:/;
1550
1551     my $opts = '';
1552     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1553     foreach my $o (@qemu_drive_options) {
1554         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1555     }
1556
1557     # snapshot only accepts on|off
1558     if (defined($drive->{snapshot})) {
1559         my $v = $drive->{snapshot} ? 'on' : 'off';
1560         $opts .= ",snapshot=$v";
1561     }
1562
1563     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1564         my ($dir, $qmpname) = @$type;
1565         if (my $v = $drive->{"mbps$dir"}) {
1566             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1567         }
1568         if (my $v = $drive->{"mbps${dir}_max"}) {
1569             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1570         }
1571         if (my $v = $drive->{"bps${dir}_max_length"}) {
1572             $opts .= ",throttling.bps$qmpname-max-length=$v";
1573         }
1574         if (my $v = $drive->{"iops${dir}"}) {
1575             $opts .= ",throttling.iops$qmpname=$v";
1576         }
1577         if (my $v = $drive->{"iops${dir}_max"}) {
1578             $opts .= ",throttling.iops$qmpname-max=$v";
1579         }
1580         if (my $v = $drive->{"iops${dir}_max_length"}) {
1581             $opts .= ",throttling.iops$qmpname-max-length=$v";
1582         }
1583     }
1584
1585     if ($pbs_name) {
1586         $format = "rbd" if $is_rbd;
1587         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1588             if !$format;
1589         $opts .= ",format=alloc-track,file.driver=$format";
1590     } elsif ($format) {
1591         $opts .= ",format=$format";
1592     }
1593
1594     my $cache_direct = 0;
1595
1596     if (my $cache = $drive->{cache}) {
1597         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1598     } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1599         $opts .= ",cache=none";
1600         $cache_direct = 1;
1601     }
1602
1603     # io_uring with cache mode writeback or writethrough on krbd will hang...
1604     my $rbd_no_io_uring = $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1605
1606     if (!$drive->{aio}) {
1607         if ($io_uring && !$rbd_no_io_uring) {
1608             # io_uring supports all cache modes
1609             $opts .= ",aio=io_uring";
1610         } else {
1611             # aio native works only with O_DIRECT
1612             if($cache_direct) {
1613                 $opts .= ",aio=native";
1614             } else {
1615                 $opts .= ",aio=threads";
1616             }
1617         }
1618     }
1619
1620     if (!drive_is_cdrom($drive)) {
1621         my $detectzeroes;
1622         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1623             $detectzeroes = 'off';
1624         } elsif ($drive->{discard}) {
1625             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1626         } else {
1627             # This used to be our default with discard not being specified:
1628             $detectzeroes = 'on';
1629         }
1630
1631         # note: 'detect-zeroes' works per blockdev and we want it to persist
1632         # after the alloc-track is removed, so put it on 'file' directly
1633         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1634         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1635     }
1636
1637     if ($pbs_name) {
1638         $opts .= ",backing=$pbs_name";
1639         $opts .= ",auto-remove=on";
1640     }
1641
1642     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1643     my $file_param = "file";
1644     if ($pbs_name) {
1645         # non-rbd drivers require the underlying file to be a seperate block
1646         # node, so add a second .file indirection
1647         $file_param .= ".file" if !$is_rbd;
1648         $file_param .= ".filename";
1649     }
1650     my $pathinfo = $path ? "$file_param=$path," : '';
1651
1652     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1653 }
1654
1655 sub print_pbs_blockdev {
1656     my ($pbs_conf, $pbs_name) = @_;
1657     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1658     $blockdev .= ",repository=$pbs_conf->{repository}";
1659     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1660     $blockdev .= ",archive=$pbs_conf->{archive}";
1661     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1662     return $blockdev;
1663 }
1664
1665 sub print_netdevice_full {
1666     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1667
1668     my $device = $net->{model};
1669     if ($net->{model} eq 'virtio') {
1670          $device = 'virtio-net-pci';
1671      };
1672
1673     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1674     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1675     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1676         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1677         # and out of each queue plus one config interrupt and control vector queue
1678         my $vectors = $net->{queues} * 2 + 2;
1679         $tmpstr .= ",vectors=$vectors,mq=on";
1680     }
1681     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1682
1683     if (my $mtu = $net->{mtu}) {
1684         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1685             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1686             if ($mtu == 1) {
1687                  $mtu = $bridge_mtu;
1688             } elsif ($mtu < 576) {
1689                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1690             } elsif ($mtu > $bridge_mtu) {
1691                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1692             }
1693             $tmpstr .= ",host_mtu=$mtu";
1694         } else {
1695             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1696         }
1697     }
1698
1699     if ($use_old_bios_files) {
1700         my $romfile;
1701         if ($device eq 'virtio-net-pci') {
1702             $romfile = 'pxe-virtio.rom';
1703         } elsif ($device eq 'e1000') {
1704             $romfile = 'pxe-e1000.rom';
1705         } elsif ($device eq 'ne2k') {
1706             $romfile = 'pxe-ne2k_pci.rom';
1707         } elsif ($device eq 'pcnet') {
1708             $romfile = 'pxe-pcnet.rom';
1709         } elsif ($device eq 'rtl8139') {
1710             $romfile = 'pxe-rtl8139.rom';
1711         }
1712         $tmpstr .= ",romfile=$romfile" if $romfile;
1713     }
1714
1715     return $tmpstr;
1716 }
1717
1718 sub print_netdev_full {
1719     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1720
1721     my $i = '';
1722     if ($netid =~ m/^net(\d+)$/) {
1723         $i = int($1);
1724     }
1725
1726     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1727
1728     my $ifname = "tap${vmid}i$i";
1729
1730     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1731     die "interface name '$ifname' is too long (max 15 character)\n"
1732         if length($ifname) >= 16;
1733
1734     my $vhostparam = '';
1735     if (is_native($arch)) {
1736         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1737     }
1738
1739     my $vmname = $conf->{name} || "vm$vmid";
1740
1741     my $netdev = "";
1742     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1743
1744     if ($net->{bridge}) {
1745         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1746             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1747     } else {
1748         $netdev = "type=user,id=$netid,hostname=$vmname";
1749     }
1750
1751     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1752
1753     return $netdev;
1754 }
1755
1756 my $vga_map = {
1757     'cirrus' => 'cirrus-vga',
1758     'std' => 'VGA',
1759     'vmware' => 'vmware-svga',
1760     'virtio' => 'virtio-vga',
1761 };
1762
1763 sub print_vga_device {
1764     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1765
1766     my $type = $vga_map->{$vga->{type}};
1767     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1768         $type = 'virtio-gpu';
1769     }
1770     my $vgamem_mb = $vga->{memory};
1771
1772     my $max_outputs = '';
1773     if ($qxlnum) {
1774         $type = $id ? 'qxl' : 'qxl-vga';
1775
1776         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1777             # set max outputs so linux can have up to 4 qxl displays with one device
1778             if (min_version($machine_version, 4, 1)) {
1779                 $max_outputs = ",max_outputs=4";
1780             }
1781         }
1782     }
1783
1784     die "no devicetype for $vga->{type}\n" if !$type;
1785
1786     my $memory = "";
1787     if ($vgamem_mb) {
1788         if ($vga->{type} eq 'virtio') {
1789             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1790             $memory = ",max_hostmem=$bytes";
1791         } elsif ($qxlnum) {
1792             # from https://www.spice-space.org/multiple-monitors.html
1793             $memory = ",vgamem_mb=$vga->{memory}";
1794             my $ram = $vgamem_mb * 4;
1795             my $vram = $vgamem_mb * 2;
1796             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1797         } else {
1798             $memory = ",vgamem_mb=$vga->{memory}";
1799         }
1800     } elsif ($qxlnum && $id) {
1801         $memory = ",ram_size=67108864,vram_size=33554432";
1802     }
1803
1804     my $edidoff = "";
1805     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1806         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1807     }
1808
1809     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1810     my $vgaid = "vga" . ($id // '');
1811     my $pciaddr;
1812
1813     if ($q35 && $vgaid eq 'vga') {
1814         # the first display uses pcie.0 bus on q35 machines
1815         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1816     } else {
1817         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1818     }
1819
1820     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1821 }
1822
1823 sub parse_number_sets {
1824     my ($set) = @_;
1825     my $res = [];
1826     foreach my $part (split(/;/, $set)) {
1827         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1828             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1829             push @$res, [ $1, $2 ];
1830         } else {
1831             die "invalid range: $part\n";
1832         }
1833     }
1834     return $res;
1835 }
1836
1837 sub parse_numa {
1838     my ($data) = @_;
1839
1840     my $res = parse_property_string($numa_fmt, $data);
1841     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1842     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1843     return $res;
1844 }
1845
1846 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1847 sub parse_net {
1848     my ($data) = @_;
1849
1850     my $res = eval { parse_property_string($net_fmt, $data) };
1851     if ($@) {
1852         warn $@;
1853         return;
1854     }
1855     if (!defined($res->{macaddr})) {
1856         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1857         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1858     }
1859     return $res;
1860 }
1861
1862 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1863 sub parse_ipconfig {
1864     my ($data) = @_;
1865
1866     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1867     if ($@) {
1868         warn $@;
1869         return;
1870     }
1871
1872     if ($res->{gw} && !$res->{ip}) {
1873         warn 'gateway specified without specifying an IP address';
1874         return;
1875     }
1876     if ($res->{gw6} && !$res->{ip6}) {
1877         warn 'IPv6 gateway specified without specifying an IPv6 address';
1878         return;
1879     }
1880     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1881         warn 'gateway specified together with DHCP';
1882         return;
1883     }
1884     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1885         # gw6 + auto/dhcp
1886         warn "IPv6 gateway specified together with $res->{ip6} address";
1887         return;
1888     }
1889
1890     if (!$res->{ip} && !$res->{ip6}) {
1891         return { ip => 'dhcp', ip6 => 'dhcp' };
1892     }
1893
1894     return $res;
1895 }
1896
1897 sub print_net {
1898     my $net = shift;
1899
1900     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1901 }
1902
1903 sub add_random_macs {
1904     my ($settings) = @_;
1905
1906     foreach my $opt (keys %$settings) {
1907         next if $opt !~ m/^net(\d+)$/;
1908         my $net = parse_net($settings->{$opt});
1909         next if !$net;
1910         $settings->{$opt} = print_net($net);
1911     }
1912 }
1913
1914 sub vm_is_volid_owner {
1915     my ($storecfg, $vmid, $volid) = @_;
1916
1917     if ($volid !~  m|^/|) {
1918         my ($path, $owner);
1919         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1920         if ($owner && ($owner == $vmid)) {
1921             return 1;
1922         }
1923     }
1924
1925     return;
1926 }
1927
1928 sub vmconfig_register_unused_drive {
1929     my ($storecfg, $vmid, $conf, $drive) = @_;
1930
1931     if (drive_is_cloudinit($drive)) {
1932         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1933         warn $@ if $@;
1934     } elsif (!drive_is_cdrom($drive)) {
1935         my $volid = $drive->{file};
1936         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1937             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1938         }
1939     }
1940 }
1941
1942 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1943 my $smbios1_fmt = {
1944     uuid => {
1945         type => 'string',
1946         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1947         format_description => 'UUID',
1948         description => "Set SMBIOS1 UUID.",
1949         optional => 1,
1950     },
1951     version => {
1952         type => 'string',
1953         pattern => '[A-Za-z0-9+\/]+={0,2}',
1954         format_description => 'Base64 encoded string',
1955         description => "Set SMBIOS1 version.",
1956         optional => 1,
1957     },
1958     serial => {
1959         type => 'string',
1960         pattern => '[A-Za-z0-9+\/]+={0,2}',
1961         format_description => 'Base64 encoded string',
1962         description => "Set SMBIOS1 serial number.",
1963         optional => 1,
1964     },
1965     manufacturer => {
1966         type => 'string',
1967         pattern => '[A-Za-z0-9+\/]+={0,2}',
1968         format_description => 'Base64 encoded string',
1969         description => "Set SMBIOS1 manufacturer.",
1970         optional => 1,
1971     },
1972     product => {
1973         type => 'string',
1974         pattern => '[A-Za-z0-9+\/]+={0,2}',
1975         format_description => 'Base64 encoded string',
1976         description => "Set SMBIOS1 product ID.",
1977         optional => 1,
1978     },
1979     sku => {
1980         type => 'string',
1981         pattern => '[A-Za-z0-9+\/]+={0,2}',
1982         format_description => 'Base64 encoded string',
1983         description => "Set SMBIOS1 SKU string.",
1984         optional => 1,
1985     },
1986     family => {
1987         type => 'string',
1988         pattern => '[A-Za-z0-9+\/]+={0,2}',
1989         format_description => 'Base64 encoded string',
1990         description => "Set SMBIOS1 family string.",
1991         optional => 1,
1992     },
1993     base64 => {
1994         type => 'boolean',
1995         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
1996         optional => 1,
1997     },
1998 };
1999
2000 sub parse_smbios1 {
2001     my ($data) = @_;
2002
2003     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2004     warn $@ if $@;
2005     return $res;
2006 }
2007
2008 sub print_smbios1 {
2009     my ($smbios1) = @_;
2010     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2011 }
2012
2013 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2014
2015 sub parse_watchdog {
2016     my ($value) = @_;
2017
2018     return if !$value;
2019
2020     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2021     warn $@ if $@;
2022     return $res;
2023 }
2024
2025 sub parse_guest_agent {
2026     my ($conf) = @_;
2027
2028     return {} if !defined($conf->{agent});
2029
2030     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2031     warn $@ if $@;
2032
2033     # if the agent is disabled ignore the other potentially set properties
2034     return {} if !$res->{enabled};
2035     return $res;
2036 }
2037
2038 sub get_qga_key {
2039     my ($conf, $key) = @_;
2040     return undef if !defined($conf->{agent});
2041
2042     my $agent = parse_guest_agent($conf);
2043     return $agent->{$key};
2044 }
2045
2046 sub parse_vga {
2047     my ($value) = @_;
2048
2049     return {} if !$value;
2050     my $res = eval { parse_property_string($vga_fmt, $value) };
2051     warn $@ if $@;
2052     return $res;
2053 }
2054
2055 sub parse_rng {
2056     my ($value) = @_;
2057
2058     return if !$value;
2059
2060     my $res = eval { parse_property_string($rng_fmt, $value) };
2061     warn $@ if $@;
2062     return $res;
2063 }
2064
2065 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2066 sub verify_usb_device {
2067     my ($value, $noerr) = @_;
2068
2069     return $value if parse_usb_device($value);
2070
2071     return if $noerr;
2072
2073     die "unable to parse usb device\n";
2074 }
2075
2076 # add JSON properties for create and set function
2077 sub json_config_properties {
2078     my $prop = shift;
2079
2080     foreach my $opt (keys %$confdesc) {
2081         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' ||
2082             $opt eq 'runningmachine' || $opt eq 'runningcpu';
2083         $prop->{$opt} = $confdesc->{$opt};
2084     }
2085
2086     return $prop;
2087 }
2088
2089 # return copy of $confdesc_cloudinit to generate documentation
2090 sub cloudinit_config_properties {
2091
2092     return dclone($confdesc_cloudinit);
2093 }
2094
2095 sub check_type {
2096     my ($key, $value) = @_;
2097
2098     die "unknown setting '$key'\n" if !$confdesc->{$key};
2099
2100     my $type = $confdesc->{$key}->{type};
2101
2102     if (!defined($value)) {
2103         die "got undefined value\n";
2104     }
2105
2106     if ($value =~ m/[\n\r]/) {
2107         die "property contains a line feed\n";
2108     }
2109
2110     if ($type eq 'boolean') {
2111         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2112         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2113         die "type check ('boolean') failed - got '$value'\n";
2114     } elsif ($type eq 'integer') {
2115         return int($1) if $value =~ m/^(\d+)$/;
2116         die "type check ('integer') failed - got '$value'\n";
2117     } elsif ($type eq 'number') {
2118         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2119         die "type check ('number') failed - got '$value'\n";
2120     } elsif ($type eq 'string') {
2121         if (my $fmt = $confdesc->{$key}->{format}) {
2122             PVE::JSONSchema::check_format($fmt, $value);
2123             return $value;
2124         }
2125         $value =~ s/^\"(.*)\"$/$1/;
2126         return $value;
2127     } else {
2128         die "internal error"
2129     }
2130 }
2131
2132 sub destroy_vm {
2133     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2134
2135     my $conf = PVE::QemuConfig->load_config($vmid);
2136
2137     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2138
2139     if ($conf->{template}) {
2140         # check if any base image is still used by a linked clone
2141         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2142                 my ($ds, $drive) = @_;
2143                 return if drive_is_cdrom($drive);
2144
2145                 my $volid = $drive->{file};
2146                 return if !$volid || $volid =~ m|^/|;
2147
2148                 die "base volume '$volid' is still in use by linked cloned\n"
2149                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2150
2151         });
2152     }
2153
2154     my $remove_owned_drive = sub {
2155         my ($ds, $drive) = @_;
2156         return if drive_is_cdrom($drive, 1);
2157
2158         my $volid = $drive->{file};
2159         return if !$volid || $volid =~ m|^/|;
2160
2161         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2162         return if !$path || !$owner || ($owner != $vmid);
2163
2164         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2165         warn "Could not remove disk '$volid', check manually: $@" if $@;
2166     };
2167
2168     # only remove disks owned by this VM (referenced in the config)
2169     my $include_opts = {
2170         include_unused => 1,
2171         extra_keys => ['vmstate'],
2172     };
2173     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2174
2175     for my $snap (values %{$conf->{snapshots}}) {
2176         next if !defined($snap->{vmstate});
2177         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2178         next if !defined($drive);
2179         $remove_owned_drive->('vmstate', $drive);
2180     }
2181
2182     if ($purge_unreferenced) { # also remove unreferenced disk
2183         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2184         PVE::Storage::foreach_volid($vmdisks, sub {
2185             my ($volid, $sid, $volname, $d) = @_;
2186             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2187             warn $@ if $@;
2188         });
2189     }
2190
2191     if (defined $replacement_conf) {
2192         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2193     } else {
2194         PVE::QemuConfig->destroy_config($vmid);
2195     }
2196 }
2197
2198 sub parse_vm_config {
2199     my ($filename, $raw) = @_;
2200
2201     return if !defined($raw);
2202
2203     my $res = {
2204         digest => Digest::SHA::sha1_hex($raw),
2205         snapshots => {},
2206         pending => {},
2207     };
2208
2209     $filename =~ m|/qemu-server/(\d+)\.conf$|
2210         || die "got strange filename '$filename'";
2211
2212     my $vmid = $1;
2213
2214     my $conf = $res;
2215     my $descr;
2216     my $section = '';
2217
2218     my @lines = split(/\n/, $raw);
2219     foreach my $line (@lines) {
2220         next if $line =~ m/^\s*$/;
2221
2222         if ($line =~ m/^\[PENDING\]\s*$/i) {
2223             $section = 'pending';
2224             if (defined($descr)) {
2225                 $descr =~ s/\s+$//;
2226                 $conf->{description} = $descr;
2227             }
2228             $descr = undef;
2229             $conf = $res->{$section} = {};
2230             next;
2231
2232         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2233             $section = $1;
2234             if (defined($descr)) {
2235                 $descr =~ s/\s+$//;
2236                 $conf->{description} = $descr;
2237             }
2238             $descr = undef;
2239             $conf = $res->{snapshots}->{$section} = {};
2240             next;
2241         }
2242
2243         if ($line =~ m/^\#(.*)\s*$/) {
2244             $descr = '' if !defined($descr);
2245             $descr .= PVE::Tools::decode_text($1) . "\n";
2246             next;
2247         }
2248
2249         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2250             $descr = '' if !defined($descr);
2251             $descr .= PVE::Tools::decode_text($2);
2252         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2253             $conf->{snapstate} = $1;
2254         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2255             my $key = $1;
2256             my $value = $2;
2257             $conf->{$key} = $value;
2258         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2259             my $value = $1;
2260             if ($section eq 'pending') {
2261                 $conf->{delete} = $value; # we parse this later
2262             } else {
2263                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2264             }
2265         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2266             my $key = $1;
2267             my $value = $2;
2268             eval { $value = check_type($key, $value); };
2269             if ($@) {
2270                 warn "vm $vmid - unable to parse value of '$key' - $@";
2271             } else {
2272                 $key = 'ide2' if $key eq 'cdrom';
2273                 my $fmt = $confdesc->{$key}->{format};
2274                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2275                     my $v = parse_drive($key, $value);
2276                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2277                         $v->{file} = $volid;
2278                         $value = print_drive($v);
2279                     } else {
2280                         warn "vm $vmid - unable to parse value of '$key'\n";
2281                         next;
2282                     }
2283                 }
2284
2285                 $conf->{$key} = $value;
2286             }
2287         } else {
2288             warn "vm $vmid - unable to parse config: $line\n";
2289         }
2290     }
2291
2292     if (defined($descr)) {
2293         $descr =~ s/\s+$//;
2294         $conf->{description} = $descr;
2295     }
2296     delete $res->{snapstate}; # just to be sure
2297
2298     return $res;
2299 }
2300
2301 sub write_vm_config {
2302     my ($filename, $conf) = @_;
2303
2304     delete $conf->{snapstate}; # just to be sure
2305
2306     if ($conf->{cdrom}) {
2307         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2308         $conf->{ide2} = $conf->{cdrom};
2309         delete $conf->{cdrom};
2310     }
2311
2312     # we do not use 'smp' any longer
2313     if ($conf->{sockets}) {
2314         delete $conf->{smp};
2315     } elsif ($conf->{smp}) {
2316         $conf->{sockets} = $conf->{smp};
2317         delete $conf->{cores};
2318         delete $conf->{smp};
2319     }
2320
2321     my $used_volids = {};
2322
2323     my $cleanup_config = sub {
2324         my ($cref, $pending, $snapname) = @_;
2325
2326         foreach my $key (keys %$cref) {
2327             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2328                 $key eq 'snapstate' || $key eq 'pending';
2329             my $value = $cref->{$key};
2330             if ($key eq 'delete') {
2331                 die "propertry 'delete' is only allowed in [PENDING]\n"
2332                     if !$pending;
2333                 # fixme: check syntax?
2334                 next;
2335             }
2336             eval { $value = check_type($key, $value); };
2337             die "unable to parse value of '$key' - $@" if $@;
2338
2339             $cref->{$key} = $value;
2340
2341             if (!$snapname && is_valid_drivename($key)) {
2342                 my $drive = parse_drive($key, $value);
2343                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2344             }
2345         }
2346     };
2347
2348     &$cleanup_config($conf);
2349
2350     &$cleanup_config($conf->{pending}, 1);
2351
2352     foreach my $snapname (keys %{$conf->{snapshots}}) {
2353         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2354         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2355     }
2356
2357     # remove 'unusedX' settings if we re-add a volume
2358     foreach my $key (keys %$conf) {
2359         my $value = $conf->{$key};
2360         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2361             delete $conf->{$key};
2362         }
2363     }
2364
2365     my $generate_raw_config = sub {
2366         my ($conf, $pending) = @_;
2367
2368         my $raw = '';
2369
2370         # add description as comment to top of file
2371         if (defined(my $descr = $conf->{description})) {
2372             if ($descr) {
2373                 foreach my $cl (split(/\n/, $descr)) {
2374                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2375                 }
2376             } else {
2377                 $raw .= "#\n" if $pending;
2378             }
2379         }
2380
2381         foreach my $key (sort keys %$conf) {
2382             next if $key =~ /^(digest|description|pending|snapshots)$/;
2383             $raw .= "$key: $conf->{$key}\n";
2384         }
2385         return $raw;
2386     };
2387
2388     my $raw = &$generate_raw_config($conf);
2389
2390     if (scalar(keys %{$conf->{pending}})){
2391         $raw .= "\n[PENDING]\n";
2392         $raw .= &$generate_raw_config($conf->{pending}, 1);
2393     }
2394
2395     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2396         $raw .= "\n[$snapname]\n";
2397         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2398     }
2399
2400     return $raw;
2401 }
2402
2403 sub load_defaults {
2404
2405     my $res = {};
2406
2407     # we use static defaults from our JSON schema configuration
2408     foreach my $key (keys %$confdesc) {
2409         if (defined(my $default = $confdesc->{$key}->{default})) {
2410             $res->{$key} = $default;
2411         }
2412     }
2413
2414     return $res;
2415 }
2416
2417 sub config_list {
2418     my $vmlist = PVE::Cluster::get_vmlist();
2419     my $res = {};
2420     return $res if !$vmlist || !$vmlist->{ids};
2421     my $ids = $vmlist->{ids};
2422     my $nodename = nodename();
2423
2424     foreach my $vmid (keys %$ids) {
2425         my $d = $ids->{$vmid};
2426         next if !$d->{node} || $d->{node} ne $nodename;
2427         next if !$d->{type} || $d->{type} ne 'qemu';
2428         $res->{$vmid}->{exists} = 1;
2429     }
2430     return $res;
2431 }
2432
2433 # test if VM uses local resources (to prevent migration)
2434 sub check_local_resources {
2435     my ($conf, $noerr) = @_;
2436
2437     my @loc_res = ();
2438
2439     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2440     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2441
2442     push @loc_res, "ivshmem" if $conf->{ivshmem};
2443
2444     foreach my $k (keys %$conf) {
2445         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2446         # sockets are safe: they will recreated be on the target side post-migrate
2447         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2448         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2449     }
2450
2451     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2452
2453     return \@loc_res;
2454 }
2455
2456 # check if used storages are available on all nodes (use by migrate)
2457 sub check_storage_availability {
2458     my ($storecfg, $conf, $node) = @_;
2459
2460     PVE::QemuConfig->foreach_volume($conf, sub {
2461         my ($ds, $drive) = @_;
2462
2463         my $volid = $drive->{file};
2464         return if !$volid;
2465
2466         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2467         return if !$sid;
2468
2469         # check if storage is available on both nodes
2470         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2471         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2472
2473         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2474
2475         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2476             if !$scfg->{content}->{$vtype};
2477    });
2478 }
2479
2480 # list nodes where all VM images are available (used by has_feature API)
2481 sub shared_nodes {
2482     my ($conf, $storecfg) = @_;
2483
2484     my $nodelist = PVE::Cluster::get_nodelist();
2485     my $nodehash = { map { $_ => 1 } @$nodelist };
2486     my $nodename = nodename();
2487
2488     PVE::QemuConfig->foreach_volume($conf, sub {
2489         my ($ds, $drive) = @_;
2490
2491         my $volid = $drive->{file};
2492         return if !$volid;
2493
2494         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2495         if ($storeid) {
2496             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2497             if ($scfg->{disable}) {
2498                 $nodehash = {};
2499             } elsif (my $avail = $scfg->{nodes}) {
2500                 foreach my $node (keys %$nodehash) {
2501                     delete $nodehash->{$node} if !$avail->{$node};
2502                 }
2503             } elsif (!$scfg->{shared}) {
2504                 foreach my $node (keys %$nodehash) {
2505                     delete $nodehash->{$node} if $node ne $nodename
2506                 }
2507             }
2508         }
2509     });
2510
2511     return $nodehash
2512 }
2513
2514 sub check_local_storage_availability {
2515     my ($conf, $storecfg) = @_;
2516
2517     my $nodelist = PVE::Cluster::get_nodelist();
2518     my $nodehash = { map { $_ => {} } @$nodelist };
2519
2520     PVE::QemuConfig->foreach_volume($conf, sub {
2521         my ($ds, $drive) = @_;
2522
2523         my $volid = $drive->{file};
2524         return if !$volid;
2525
2526         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2527         if ($storeid) {
2528             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2529
2530             if ($scfg->{disable}) {
2531                 foreach my $node (keys %$nodehash) {
2532                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2533                 }
2534             } elsif (my $avail = $scfg->{nodes}) {
2535                 foreach my $node (keys %$nodehash) {
2536                     if (!$avail->{$node}) {
2537                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2538                     }
2539                 }
2540             }
2541         }
2542     });
2543
2544     foreach my $node (values %$nodehash) {
2545         if (my $unavail = $node->{unavailable_storages}) {
2546             $node->{unavailable_storages} = [ sort keys %$unavail ];
2547         }
2548     }
2549
2550     return $nodehash
2551 }
2552
2553 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2554 sub check_running {
2555     my ($vmid, $nocheck, $node) = @_;
2556
2557     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2558     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2559 }
2560
2561 sub vzlist {
2562
2563     my $vzlist = config_list();
2564
2565     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2566
2567     while (defined(my $de = $fd->read)) {
2568         next if $de !~ m/^(\d+)\.pid$/;
2569         my $vmid = $1;
2570         next if !defined($vzlist->{$vmid});
2571         if (my $pid = check_running($vmid)) {
2572             $vzlist->{$vmid}->{pid} = $pid;
2573         }
2574     }
2575
2576     return $vzlist;
2577 }
2578
2579 our $vmstatus_return_properties = {
2580     vmid => get_standard_option('pve-vmid'),
2581     status => {
2582         description => "Qemu process status.",
2583         type => 'string',
2584         enum => ['stopped', 'running'],
2585     },
2586     maxmem => {
2587         description => "Maximum memory in bytes.",
2588         type => 'integer',
2589         optional => 1,
2590         renderer => 'bytes',
2591     },
2592     maxdisk => {
2593         description => "Root disk size in bytes.",
2594         type => 'integer',
2595         optional => 1,
2596         renderer => 'bytes',
2597     },
2598     name => {
2599         description => "VM name.",
2600         type => 'string',
2601         optional => 1,
2602     },
2603     qmpstatus => {
2604         description => "Qemu QMP agent status.",
2605         type => 'string',
2606         optional => 1,
2607     },
2608     pid => {
2609         description => "PID of running qemu process.",
2610         type => 'integer',
2611         optional => 1,
2612     },
2613     uptime => {
2614         description => "Uptime.",
2615         type => 'integer',
2616         optional => 1,
2617         renderer => 'duration',
2618     },
2619     cpus => {
2620         description => "Maximum usable CPUs.",
2621         type => 'number',
2622         optional => 1,
2623     },
2624     lock => {
2625         description => "The current config lock, if any.",
2626         type => 'string',
2627         optional => 1,
2628     },
2629     tags => {
2630         description  => "The current configured tags, if any",
2631         type => 'string',
2632         optional => 1,
2633     },
2634     'running-machine' => {
2635         description => "The currently running machine type (if running).",
2636         type => 'string',
2637         optional => 1,
2638     },
2639     'running-qemu' => {
2640         description => "The currently running QEMU version (if running).",
2641         type => 'string',
2642         optional => 1,
2643     },
2644 };
2645
2646 my $last_proc_pid_stat;
2647
2648 # get VM status information
2649 # This must be fast and should not block ($full == false)
2650 # We only query KVM using QMP if $full == true (this can be slow)
2651 sub vmstatus {
2652     my ($opt_vmid, $full) = @_;
2653
2654     my $res = {};
2655
2656     my $storecfg = PVE::Storage::config();
2657
2658     my $list = vzlist();
2659     my $defaults = load_defaults();
2660
2661     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2662
2663     my $cpucount = $cpuinfo->{cpus} || 1;
2664
2665     foreach my $vmid (keys %$list) {
2666         next if $opt_vmid && ($vmid ne $opt_vmid);
2667
2668         my $conf = PVE::QemuConfig->load_config($vmid);
2669
2670         my $d = { vmid => int($vmid) };
2671         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2672
2673         # fixme: better status?
2674         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2675
2676         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2677         if (defined($size)) {
2678             $d->{disk} = 0; # no info available
2679             $d->{maxdisk} = $size;
2680         } else {
2681             $d->{disk} = 0;
2682             $d->{maxdisk} = 0;
2683         }
2684
2685         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2686             * ($conf->{cores} || $defaults->{cores});
2687         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2688         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2689
2690         $d->{name} = $conf->{name} || "VM $vmid";
2691         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2692             : $defaults->{memory}*(1024*1024);
2693
2694         if ($conf->{balloon}) {
2695             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2696             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2697                 : $defaults->{shares};
2698         }
2699
2700         $d->{uptime} = 0;
2701         $d->{cpu} = 0;
2702         $d->{mem} = 0;
2703
2704         $d->{netout} = 0;
2705         $d->{netin} = 0;
2706
2707         $d->{diskread} = 0;
2708         $d->{diskwrite} = 0;
2709
2710         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2711
2712         $d->{serial} = 1 if conf_has_serial($conf);
2713         $d->{lock} = $conf->{lock} if $conf->{lock};
2714         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2715
2716         $res->{$vmid} = $d;
2717     }
2718
2719     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2720     foreach my $dev (keys %$netdev) {
2721         next if $dev !~ m/^tap([1-9]\d*)i/;
2722         my $vmid = $1;
2723         my $d = $res->{$vmid};
2724         next if !$d;
2725
2726         $d->{netout} += $netdev->{$dev}->{receive};
2727         $d->{netin} += $netdev->{$dev}->{transmit};
2728
2729         if ($full) {
2730             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2731             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2732         }
2733
2734     }
2735
2736     my $ctime = gettimeofday;
2737
2738     foreach my $vmid (keys %$list) {
2739
2740         my $d = $res->{$vmid};
2741         my $pid = $d->{pid};
2742         next if !$pid;
2743
2744         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2745         next if !$pstat; # not running
2746
2747         my $used = $pstat->{utime} + $pstat->{stime};
2748
2749         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2750
2751         if ($pstat->{vsize}) {
2752             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2753         }
2754
2755         my $old = $last_proc_pid_stat->{$pid};
2756         if (!$old) {
2757             $last_proc_pid_stat->{$pid} = {
2758                 time => $ctime,
2759                 used => $used,
2760                 cpu => 0,
2761             };
2762             next;
2763         }
2764
2765         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2766
2767         if ($dtime > 1000) {
2768             my $dutime = $used -  $old->{used};
2769
2770             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2771             $last_proc_pid_stat->{$pid} = {
2772                 time => $ctime,
2773                 used => $used,
2774                 cpu => $d->{cpu},
2775             };
2776         } else {
2777             $d->{cpu} = $old->{cpu};
2778         }
2779     }
2780
2781     return $res if !$full;
2782
2783     my $qmpclient = PVE::QMPClient->new();
2784
2785     my $ballooncb = sub {
2786         my ($vmid, $resp) = @_;
2787
2788         my $info = $resp->{'return'};
2789         return if !$info->{max_mem};
2790
2791         my $d = $res->{$vmid};
2792
2793         # use memory assigned to VM
2794         $d->{maxmem} = $info->{max_mem};
2795         $d->{balloon} = $info->{actual};
2796
2797         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2798             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2799             $d->{freemem} = $info->{free_mem};
2800         }
2801
2802         $d->{ballooninfo} = $info;
2803     };
2804
2805     my $blockstatscb = sub {
2806         my ($vmid, $resp) = @_;
2807         my $data = $resp->{'return'} || [];
2808         my $totalrdbytes = 0;
2809         my $totalwrbytes = 0;
2810
2811         for my $blockstat (@$data) {
2812             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2813             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2814
2815             $blockstat->{device} =~ s/drive-//;
2816             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2817         }
2818         $res->{$vmid}->{diskread} = $totalrdbytes;
2819         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2820     };
2821
2822     my $machinecb = sub {
2823         my ($vmid, $resp) = @_;
2824         my $data = $resp->{'return'} || [];
2825
2826         $res->{$vmid}->{'running-machine'} =
2827             PVE::QemuServer::Machine::current_from_query_machines($data);
2828     };
2829
2830     my $versioncb = sub {
2831         my ($vmid, $resp) = @_;
2832         my $data = $resp->{'return'} // {};
2833         my $version = 'unknown';
2834
2835         if (my $v = $data->{qemu}) {
2836             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
2837         }
2838
2839         $res->{$vmid}->{'running-qemu'} = $version;
2840     };
2841
2842     my $statuscb = sub {
2843         my ($vmid, $resp) = @_;
2844
2845         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2846         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
2847         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
2848         # this fails if ballon driver is not loaded, so this must be
2849         # the last commnand (following command are aborted if this fails).
2850         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2851
2852         my $status = 'unknown';
2853         if (!defined($status = $resp->{'return'}->{status})) {
2854             warn "unable to get VM status\n";
2855             return;
2856         }
2857
2858         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2859     };
2860
2861     foreach my $vmid (keys %$list) {
2862         next if $opt_vmid && ($vmid ne $opt_vmid);
2863         next if !$res->{$vmid}->{pid}; # not running
2864         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2865     }
2866
2867     $qmpclient->queue_execute(undef, 2);
2868
2869     foreach my $vmid (keys %$list) {
2870         next if $opt_vmid && ($vmid ne $opt_vmid);
2871         next if !$res->{$vmid}->{pid}; #not running
2872
2873         # we can't use the $qmpclient since it might have already aborted on
2874         # 'query-balloon', but this might also fail for older versions...
2875         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
2876         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
2877     }
2878
2879     foreach my $vmid (keys %$list) {
2880         next if $opt_vmid && ($vmid ne $opt_vmid);
2881         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2882     }
2883
2884     return $res;
2885 }
2886
2887 sub conf_has_serial {
2888     my ($conf) = @_;
2889
2890     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2891         if ($conf->{"serial$i"}) {
2892             return 1;
2893         }
2894     }
2895
2896     return 0;
2897 }
2898
2899 sub conf_has_audio {
2900     my ($conf, $id) = @_;
2901
2902     $id //= 0;
2903     my $audio = $conf->{"audio$id"};
2904     return if !defined($audio);
2905
2906     my $audioproperties = parse_property_string($audio_fmt, $audio);
2907     my $audiodriver = $audioproperties->{driver} // 'spice';
2908
2909     return {
2910         dev => $audioproperties->{device},
2911         dev_id => "audiodev$id",
2912         backend => $audiodriver,
2913         backend_id => "$audiodriver-backend${id}",
2914     };
2915 }
2916
2917 sub audio_devs {
2918     my ($audio, $audiopciaddr, $machine_version) = @_;
2919
2920     my $devs = [];
2921
2922     my $id = $audio->{dev_id};
2923     my $audiodev = "";
2924     if (min_version($machine_version, 4, 2)) {
2925         $audiodev = ",audiodev=$audio->{backend_id}";
2926     }
2927
2928     if ($audio->{dev} eq 'AC97') {
2929         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
2930     } elsif ($audio->{dev} =~ /intel\-hda$/) {
2931         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
2932         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
2933         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
2934     } else {
2935         die "unkown audio device '$audio->{dev}', implement me!";
2936     }
2937
2938     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
2939
2940     return $devs;
2941 }
2942
2943 sub vga_conf_has_spice {
2944     my ($vga) = @_;
2945
2946     my $vgaconf = parse_vga($vga);
2947     my $vgatype = $vgaconf->{type};
2948     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
2949
2950     return $1 || 1;
2951 }
2952
2953 sub is_native($) {
2954     my ($arch) = @_;
2955     return get_host_arch() eq $arch;
2956 }
2957
2958 sub get_vm_arch {
2959     my ($conf) = @_;
2960     return $conf->{arch} // get_host_arch();
2961 }
2962
2963 my $default_machines = {
2964     x86_64 => 'pc',
2965     aarch64 => 'virt',
2966 };
2967
2968 sub get_installed_machine_version {
2969     my ($kvmversion) = @_;
2970     $kvmversion = kvm_user_version() if !defined($kvmversion);
2971     $kvmversion =~ m/^(\d+\.\d+)/;
2972     return $1;
2973 }
2974
2975 sub windows_get_pinned_machine_version {
2976     my ($machine, $base_version, $kvmversion) = @_;
2977
2978     my $pin_version = $base_version;
2979     if (!defined($base_version) ||
2980         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
2981     ) {
2982         $pin_version = get_installed_machine_version($kvmversion);
2983     }
2984     if (!$machine || $machine eq 'pc') {
2985         $machine = "pc-i440fx-$pin_version";
2986     } elsif ($machine eq 'q35') {
2987         $machine = "pc-q35-$pin_version";
2988     } elsif ($machine eq 'virt') {
2989         $machine = "virt-$pin_version";
2990     } else {
2991         warn "unknown machine type '$machine', not touching that!\n";
2992     }
2993
2994     return $machine;
2995 }
2996
2997 sub get_vm_machine {
2998     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
2999
3000     my $machine = $forcemachine || $conf->{machine};
3001
3002     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3003         $kvmversion //= kvm_user_version();
3004         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3005         # layout which confuses windows quite a bit and may result in various regressions..
3006         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3007         if (windows_version($conf->{ostype})) {
3008             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3009         }
3010         $arch //= 'x86_64';
3011         $machine ||= $default_machines->{$arch};
3012         if ($add_pve_version) {
3013             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3014             $machine .= "+pve$pvever";
3015         }
3016     }
3017
3018     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3019         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3020         $machine = $1 if $is_pxe;
3021
3022         # for version-pinned machines that do not include a pve-version (e.g.
3023         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3024         $machine .= '+pve0';
3025
3026         $machine .= '.pxe' if $is_pxe;
3027     }
3028
3029     return $machine;
3030 }
3031
3032 sub get_ovmf_files($) {
3033     my ($arch) = @_;
3034
3035     my $ovmf = $OVMF->{$arch}
3036         or die "no OVMF images known for architecture '$arch'\n";
3037
3038     return @$ovmf;
3039 }
3040
3041 my $Arch2Qemu = {
3042     aarch64 => '/usr/bin/qemu-system-aarch64',
3043     x86_64 => '/usr/bin/qemu-system-x86_64',
3044 };
3045 sub get_command_for_arch($) {
3046     my ($arch) = @_;
3047     return '/usr/bin/kvm' if is_native($arch);
3048
3049     my $cmd = $Arch2Qemu->{$arch}
3050         or die "don't know how to emulate architecture '$arch'\n";
3051     return $cmd;
3052 }
3053
3054 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3055 # to use in a QEMU command line (-cpu element), first array_intersect the result
3056 # of query_supported_ with query_understood_. This is necessary because:
3057 #
3058 # a) query_understood_ returns flags the host cannot use and
3059 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3060 #    flags, but CPU settings - with most of them being flags. Those settings
3061 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3062 #
3063 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3064 # expensive. If you need the value returned from this, you can get it much
3065 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3066 # $accel being 'kvm' or 'tcg'.
3067 #
3068 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3069 # changes, automatically populating pmxcfs.
3070 #
3071 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3072 # since kvm and tcg machines support different flags
3073 #
3074 sub query_supported_cpu_flags {
3075     my ($arch) = @_;
3076
3077     $arch //= get_host_arch();
3078     my $default_machine = $default_machines->{$arch};
3079
3080     my $flags = {};
3081
3082     # FIXME: Once this is merged, the code below should work for ARM as well:
3083     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3084     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3085         $arch eq "aarch64";
3086
3087     my $kvm_supported = defined(kvm_version());
3088     my $qemu_cmd = get_command_for_arch($arch);
3089     my $fakevmid = -1;
3090     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3091
3092     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3093     my $query_supported_run_qemu = sub {
3094         my ($kvm) = @_;
3095
3096         my $flags = {};
3097         my $cmd = [
3098             $qemu_cmd,
3099             '-machine', $default_machine,
3100             '-display', 'none',
3101             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3102             '-mon', 'chardev=qmp,mode=control',
3103             '-pidfile', $pidfile,
3104             '-S', '-daemonize'
3105         ];
3106
3107         if (!$kvm) {
3108             push @$cmd, '-accel', 'tcg';
3109         }
3110
3111         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3112         die "QEMU flag querying VM exited with code " . $rc if $rc;
3113
3114         eval {
3115             my $cmd_result = mon_cmd(
3116                 $fakevmid,
3117                 'query-cpu-model-expansion',
3118                 type => 'full',
3119                 model => { name => 'host' }
3120             );
3121
3122             my $props = $cmd_result->{model}->{props};
3123             foreach my $prop (keys %$props) {
3124                 next if $props->{$prop} ne '1';
3125                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3126                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3127                 # We only keep those with underscores, to match /proc/cpuinfo
3128                 $prop =~ s/\.|-/_/g;
3129                 $flags->{$prop} = 1;
3130             }
3131         };
3132         my $err = $@;
3133
3134         # force stop with 10 sec timeout and 'nocheck'
3135         # always stop, even if QMP failed
3136         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3137
3138         die $err if $err;
3139
3140         return [ sort keys %$flags ];
3141     };
3142
3143     # We need to query QEMU twice, since KVM and TCG have different supported flags
3144     PVE::QemuConfig->lock_config($fakevmid, sub {
3145         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3146         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3147
3148         if ($kvm_supported) {
3149             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3150             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3151         }
3152     });
3153
3154     return $flags;
3155 }
3156
3157 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3158 my $understood_cpu_flag_dir = "/usr/share/kvm";
3159 sub query_understood_cpu_flags {
3160     my $arch = get_host_arch();
3161     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3162
3163     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3164         if ! -e $filepath;
3165
3166     my $raw = file_get_contents($filepath);
3167     $raw =~ s/^\s+|\s+$//g;
3168     my @flags = split(/\s+/, $raw);
3169
3170     return \@flags;
3171 }
3172
3173 sub config_to_command {
3174     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3175         $pbs_backing) = @_;
3176
3177     my $cmd = [];
3178     my $globalFlags = [];
3179     my $machineFlags = [];
3180     my $rtcFlags = [];
3181     my $devices = [];
3182     my $pciaddr = '';
3183     my $bridges = {};
3184     my $ostype = $conf->{ostype};
3185     my $winversion = windows_version($ostype);
3186     my $kvm = $conf->{kvm};
3187     my $nodename = nodename();
3188
3189     my $arch = get_vm_arch($conf);
3190     my $kvm_binary = get_command_for_arch($arch);
3191     my $kvmver = kvm_user_version($kvm_binary);
3192
3193     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3194         $kvmver //= "undefined";
3195         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3196     }
3197
3198     my $add_pve_version = min_version($kvmver, 4, 1);
3199
3200     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3201     my $machine_version = extract_version($machine_type, $kvmver);
3202     $kvm //= 1 if is_native($arch);
3203
3204     $machine_version =~ m/(\d+)\.(\d+)/;
3205     my ($machine_major, $machine_minor) = ($1, $2);
3206
3207     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3208         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3209     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3210         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3211             ." please upgrade node '$nodename'\n"
3212     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3213         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3214         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3215             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3216             ." node '$nodename'\n";
3217     }
3218
3219     # if a specific +pve version is required for a feature, use $version_guard
3220     # instead of min_version to allow machines to be run with the minimum
3221     # required version
3222     my $required_pve_version = 0;
3223     my $version_guard = sub {
3224         my ($major, $minor, $pve) = @_;
3225         return 0 if !min_version($machine_version, $major, $minor, $pve);
3226         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3227         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3228         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3229         return 1;
3230     };
3231
3232     if ($kvm && !defined kvm_version()) {
3233         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3234             ." or enable in BIOS.\n";
3235     }
3236
3237     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3238     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3239     my $use_old_bios_files = undef;
3240     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3241
3242     my $cpuunits = defined($conf->{cpuunits}) ?
3243             $conf->{cpuunits} : $defaults->{cpuunits};
3244
3245     push @$cmd, $kvm_binary;
3246
3247     push @$cmd, '-id', $vmid;
3248
3249     my $vmname = $conf->{name} || "vm$vmid";
3250
3251     push @$cmd, '-name', $vmname;
3252
3253     push @$cmd, '-no-shutdown';
3254
3255     my $use_virtio = 0;
3256
3257     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3258     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3259     push @$cmd, '-mon', "chardev=qmp,mode=control";
3260
3261     if (min_version($machine_version, 2, 12)) {
3262         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3263         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3264     }
3265
3266     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3267
3268     push @$cmd, '-daemonize';
3269
3270     if ($conf->{smbios1}) {
3271         my $smbios_conf = parse_smbios1($conf->{smbios1});
3272         if ($smbios_conf->{base64}) {
3273             # Do not pass base64 flag to qemu
3274             delete $smbios_conf->{base64};
3275             my $smbios_string = "";
3276             foreach my $key (keys %$smbios_conf) {
3277                 my $value;
3278                 if ($key eq "uuid") {
3279                     $value = $smbios_conf->{uuid}
3280                 } else {
3281                     $value = decode_base64($smbios_conf->{$key});
3282                 }
3283                 # qemu accepts any binary data, only commas need escaping by double comma
3284                 $value =~ s/,/,,/g;
3285                 $smbios_string .= "," . $key . "=" . $value if $value;
3286             }
3287             push @$cmd, '-smbios', "type=1" . $smbios_string;
3288         } else {
3289             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3290         }
3291     }
3292
3293     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3294         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3295         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3296
3297         my ($path, $format);
3298         my $read_only_str = '';
3299         if (my $efidisk = $conf->{efidisk0}) {
3300             my $d = parse_drive('efidisk0', $efidisk);
3301             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3302             $format = $d->{format};
3303             if ($storeid) {
3304                 $path = PVE::Storage::path($storecfg, $d->{file});
3305                 if (!defined($format)) {
3306                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3307                     $format = qemu_img_format($scfg, $volname);
3308                 }
3309             } else {
3310                 $path = $d->{file};
3311                 die "efidisk format must be specified\n"
3312                     if !defined($format);
3313             }
3314
3315             $read_only_str = ',readonly=on' if drive_is_read_only($conf, $d);
3316         } else {
3317             warn "no efidisk configured! Using temporary efivars disk.\n";
3318             $path = "/tmp/$vmid-ovmf.fd";
3319             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3320             $format = 'raw';
3321         }
3322
3323         my $size_str = "";
3324
3325         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3326             $size_str = ",size=" . (-s $ovmf_vars);
3327         }
3328
3329         # on slower ceph clusters, booting without cache on efidisk can take a while, see #3329
3330         my $cache = $path =~ m/^rbd:/ ? ',cache=writeback' : '';
3331
3332         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code";
3333         push @$cmd, '-drive', "if=pflash,unit=1$cache,format=$format,id=drive-efidisk0$size_str,file=${path}${read_only_str}";
3334     }
3335
3336     # load q35 config
3337     if ($q35) {
3338         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3339         if (min_version($machine_version, 4, 0)) {
3340             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3341         } else {
3342             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3343         }
3344     }
3345
3346     if ($conf->{vmgenid}) {
3347         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3348     }
3349
3350     # add usb controllers
3351     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3352         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3353     push @$devices, @usbcontrollers if @usbcontrollers;
3354     my $vga = parse_vga($conf->{vga});
3355
3356     my $qxlnum = vga_conf_has_spice($conf->{vga});
3357     $vga->{type} = 'qxl' if $qxlnum;
3358
3359     if (!$vga->{type}) {
3360         if ($arch eq 'aarch64') {
3361             $vga->{type} = 'virtio';
3362         } elsif (min_version($machine_version, 2, 9)) {
3363             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3364         } else {
3365             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3366         }
3367     }
3368
3369     # enable absolute mouse coordinates (needed by vnc)
3370     my $tablet;
3371     if (defined($conf->{tablet})) {
3372         $tablet = $conf->{tablet};
3373     } else {
3374         $tablet = $defaults->{tablet};
3375         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3376         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3377     }
3378
3379     if ($tablet) {
3380         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3381         my $kbd = print_keyboarddevice_full($conf, $arch);
3382         push @$devices, '-device', $kbd if defined($kbd);
3383     }
3384
3385     my $bootorder = device_bootorder($conf);
3386
3387     # host pci device passthrough
3388     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3389         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3390
3391     # usb devices
3392     my $usb_dev_features = {};
3393     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3394
3395     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3396         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3397     push @$devices, @usbdevices if @usbdevices;
3398
3399     # serial devices
3400     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3401         if (my $path = $conf->{"serial$i"}) {
3402             if ($path eq 'socket') {
3403                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3404                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3405                 # On aarch64, serial0 is the UART device. Qemu only allows
3406                 # connecting UART devices via the '-serial' command line, as
3407                 # the device has a fixed slot on the hardware...
3408                 if ($arch eq 'aarch64' && $i == 0) {
3409                     push @$devices, '-serial', "chardev:serial$i";
3410                 } else {
3411                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3412                 }
3413             } else {
3414                 die "no such serial device\n" if ! -c $path;
3415                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3416                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3417             }
3418         }
3419     }
3420
3421     # parallel devices
3422     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3423         if (my $path = $conf->{"parallel$i"}) {
3424             die "no such parallel device\n" if ! -c $path;
3425             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3426             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3427             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3428         }
3429     }
3430
3431     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3432         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3433         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3434         push @$devices, @$audio_devs;
3435     }
3436
3437     my $sockets = 1;
3438     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3439     $sockets = $conf->{sockets} if  $conf->{sockets};
3440
3441     my $cores = $conf->{cores} || 1;
3442
3443     my $maxcpus = $sockets * $cores;
3444
3445     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3446
3447     my $allowed_vcpus = $cpuinfo->{cpus};
3448
3449     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3450         if ($allowed_vcpus < $maxcpus);
3451
3452     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3453
3454         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3455         for (my $i = 2; $i <= $vcpus; $i++)  {
3456             my $cpustr = print_cpu_device($conf,$i);
3457             push @$cmd, '-device', $cpustr;
3458         }
3459
3460     } else {
3461
3462         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3463     }
3464     push @$cmd, '-nodefaults';
3465
3466     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3467
3468     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3469
3470     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3471
3472     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3473         push @$devices, '-device', print_vga_device(
3474             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3475         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3476         push @$cmd,  '-vnc', "unix:$socket,password=on";
3477     } else {
3478         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3479         push @$cmd, '-nographic';
3480     }
3481
3482     # time drift fix
3483     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3484     my $useLocaltime = $conf->{localtime};
3485
3486     if ($winversion >= 5) { # windows
3487         $useLocaltime = 1 if !defined($conf->{localtime});
3488
3489         # use time drift fix when acpi is enabled
3490         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3491             $tdf = 1 if !defined($conf->{tdf});
3492         }
3493     }
3494
3495     if ($winversion >= 6) {
3496         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3497         push @$cmd, '-no-hpet';
3498     }
3499
3500     push @$rtcFlags, 'driftfix=slew' if $tdf;
3501
3502     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3503         push @$rtcFlags, "base=$conf->{startdate}";
3504     } elsif ($useLocaltime) {
3505         push @$rtcFlags, 'base=localtime';
3506     }
3507
3508     if ($forcecpu) {
3509         push @$cmd, '-cpu', $forcecpu;
3510     } else {
3511         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3512     }
3513
3514     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3515
3516     push @$cmd, '-S' if $conf->{freeze};
3517
3518     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3519
3520     my $guest_agent = parse_guest_agent($conf);
3521
3522     if ($guest_agent->{enabled}) {
3523         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3524         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3525
3526         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3527             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3528             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3529             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3530         } elsif ($guest_agent->{type} eq 'isa') {
3531             push @$devices, '-device', "isa-serial,chardev=qga0";
3532         }
3533     }
3534
3535     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3536     if ($rng && $version_guard->(4, 1, 2)) {
3537         check_rng_source($rng->{source});
3538
3539         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3540         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3541         my $limiter_str = "";
3542         if ($max_bytes) {
3543             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3544         }
3545
3546         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3547         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3548         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3549     }
3550
3551     my $spice_port;
3552
3553     if ($qxlnum) {
3554         if ($qxlnum > 1) {
3555             if ($winversion){
3556                 for (my $i = 1; $i < $qxlnum; $i++){
3557                     push @$devices, '-device', print_vga_device(
3558                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3559                 }
3560             } else {
3561                 # assume other OS works like Linux
3562                 my ($ram, $vram) = ("134217728", "67108864");
3563                 if ($vga->{memory}) {
3564                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3565                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3566                 }
3567                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3568                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3569             }
3570         }
3571
3572         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3573
3574         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3575         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3576         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3577
3578         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3579         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3580         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3581
3582         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3583         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3584
3585         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3586         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3587         if ($spice_enhancement->{foldersharing}) {
3588             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3589             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3590         }
3591
3592         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3593         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3594             if $spice_enhancement->{videostreaming};
3595
3596         push @$devices, '-spice', "$spice_opts";
3597     }
3598
3599     # enable balloon by default, unless explicitly disabled
3600     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3601         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3602         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3603     }
3604
3605     if ($conf->{watchdog}) {
3606         my $wdopts = parse_watchdog($conf->{watchdog});
3607         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3608         my $watchdog = $wdopts->{model} || 'i6300esb';
3609         push @$devices, '-device', "$watchdog$pciaddr";
3610         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3611     }
3612
3613     my $vollist = [];
3614     my $scsicontroller = {};
3615     my $ahcicontroller = {};
3616     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3617
3618     # Add iscsi initiator name if available
3619     if (my $initiator = get_initiator_name()) {
3620         push @$devices, '-iscsi', "initiator-name=$initiator";
3621     }
3622
3623     PVE::QemuConfig->foreach_volume($conf, sub {
3624         my ($ds, $drive) = @_;
3625
3626         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3627             check_volume_storage_type($storecfg, $drive->{file});
3628             push @$vollist, $drive->{file};
3629         }
3630
3631         # ignore efidisk here, already added in bios/fw handling code above
3632         return if $drive->{interface} eq 'efidisk';
3633
3634         $use_virtio = 1 if $ds =~ m/^virtio/;
3635
3636         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3637
3638         if ($drive->{interface} eq 'virtio'){
3639            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3640         }
3641
3642         if ($drive->{interface} eq 'scsi') {
3643
3644             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3645
3646             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3647                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3648
3649             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3650             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3651
3652             my $iothread = '';
3653             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3654                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3655                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3656             } elsif ($drive->{iothread}) {
3657                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3658             }
3659
3660             my $queues = '';
3661             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3662                 $queues = ",num_queues=$drive->{queues}";
3663             }
3664
3665             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3666                 if !$scsicontroller->{$controller};
3667             $scsicontroller->{$controller}=1;
3668         }
3669
3670         if ($drive->{interface} eq 'sata') {
3671             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3672             $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3673             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3674                 if !$ahcicontroller->{$controller};
3675             $ahcicontroller->{$controller}=1;
3676         }
3677
3678         my $pbs_conf = $pbs_backing->{$ds};
3679         my $pbs_name = undef;
3680         if ($pbs_conf) {
3681             $pbs_name = "drive-$ds-pbs";
3682             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
3683         }
3684
3685         my $drive_cmd = print_drive_commandline_full(
3686             $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
3687
3688         # extra protection for templates, but SATA and IDE don't support it..
3689         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
3690
3691         push @$devices, '-drive',$drive_cmd;
3692         push @$devices, '-device', print_drivedevice_full(
3693             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3694     });
3695
3696     for (my $i = 0; $i < $MAX_NETS; $i++) {
3697         my $netname = "net$i";
3698
3699         next if !$conf->{$netname};
3700         my $d = parse_net($conf->{$netname});
3701         next if !$d;
3702
3703         $use_virtio = 1 if $d->{model} eq 'virtio';
3704
3705         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
3706
3707         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
3708         push @$devices, '-netdev', $netdevfull;
3709
3710         my $netdevicefull = print_netdevice_full(
3711             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
3712
3713         push @$devices, '-device', $netdevicefull;
3714     }
3715
3716     if ($conf->{ivshmem}) {
3717         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3718
3719         my $bus;
3720         if ($q35) {
3721             $bus = print_pcie_addr("ivshmem");
3722         } else {
3723             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3724         }
3725
3726         my $ivshmem_name = $ivshmem->{name} // $vmid;
3727         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3728
3729         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3730         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
3731             .",size=$ivshmem->{size}M";
3732     }
3733
3734     # pci.4 is nested in pci.1
3735     $bridges->{1} = 1 if $bridges->{4};
3736
3737     if (!$q35) {
3738         # add pci bridges
3739         if (min_version($machine_version, 2, 3)) {
3740            $bridges->{1} = 1;
3741            $bridges->{2} = 1;
3742         }
3743
3744         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3745
3746     }
3747
3748     for my $k (sort {$b cmp $a} keys %$bridges) {
3749         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3750
3751         my $k_name = $k;
3752         if ($k == 2 && $legacy_igd) {
3753             $k_name = "$k-igd";
3754         }
3755         $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
3756
3757         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3758         if ($q35) {
3759             # add after -readconfig pve-q35.cfg
3760             splice @$devices, 2, 0, '-device', $devstr;
3761         } else {
3762             unshift @$devices, '-device', $devstr if $k > 0;
3763         }
3764     }
3765
3766     if (!$kvm) {
3767         push @$machineFlags, 'accel=tcg';
3768     }
3769
3770     my $machine_type_min = $machine_type;
3771     if ($add_pve_version) {
3772         $machine_type_min =~ s/\+pve\d+$//;
3773         $machine_type_min .= "+pve$required_pve_version";
3774     }
3775     push @$machineFlags, "type=${machine_type_min}";
3776
3777     push @$cmd, @$devices;
3778     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
3779     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
3780     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
3781
3782     if (my $vmstate = $conf->{vmstate}) {
3783         my $statepath = PVE::Storage::path($storecfg, $vmstate);
3784         push @$vollist, $vmstate;
3785         push @$cmd, '-loadstate', $statepath;
3786         print "activating and using '$vmstate' as vmstate\n";
3787     }
3788
3789     if (PVE::QemuConfig->is_template($conf)) {
3790         # needed to workaround base volumes being read-only
3791         push @$cmd, '-snapshot';
3792     }
3793
3794     # add custom args
3795     if ($conf->{args}) {
3796         my $aa = PVE::Tools::split_args($conf->{args});
3797         push @$cmd, @$aa;
3798     }
3799
3800     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3801 }
3802
3803 sub check_rng_source {
3804     my ($source) = @_;
3805
3806     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3807     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
3808         if ! -e $source;
3809
3810     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
3811     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
3812         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
3813         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
3814         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
3815             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
3816             ." to the host.\n";
3817     }
3818 }
3819
3820 sub spice_port {
3821     my ($vmid) = @_;
3822
3823     my $res = mon_cmd($vmid, 'query-spice');
3824
3825     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3826 }
3827
3828 sub vm_devices_list {
3829     my ($vmid) = @_;
3830
3831     my $res = mon_cmd($vmid, 'query-pci');
3832     my $devices_to_check = [];
3833     my $devices = {};
3834     foreach my $pcibus (@$res) {
3835         push @$devices_to_check, @{$pcibus->{devices}},
3836     }
3837
3838     while (@$devices_to_check) {
3839         my $to_check = [];
3840         for my $d (@$devices_to_check) {
3841             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3842             next if !$d->{'pci_bridge'};
3843
3844             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3845             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3846         }
3847         $devices_to_check = $to_check;
3848     }
3849
3850     my $resblock = mon_cmd($vmid, 'query-block');
3851     foreach my $block (@$resblock) {
3852         if($block->{device} =~ m/^drive-(\S+)/){
3853                 $devices->{$1} = 1;
3854         }
3855     }
3856
3857     my $resmice = mon_cmd($vmid, 'query-mice');
3858     foreach my $mice (@$resmice) {
3859         if ($mice->{name} eq 'QEMU HID Tablet') {
3860             $devices->{tablet} = 1;
3861             last;
3862         }
3863     }
3864
3865     # for usb devices there is no query-usb
3866     # but we can iterate over the entries in
3867     # qom-list path=/machine/peripheral
3868     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3869     foreach my $per (@$resperipheral) {
3870         if ($per->{name} =~ m/^usb\d+$/) {
3871             $devices->{$per->{name}} = 1;
3872         }
3873     }
3874
3875     return $devices;
3876 }
3877
3878 sub vm_deviceplug {
3879     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3880
3881     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3882
3883     my $devices_list = vm_devices_list($vmid);
3884     return 1 if defined($devices_list->{$deviceid});
3885
3886     # add PCI bridge if we need it for the device
3887     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
3888
3889     if ($deviceid eq 'tablet') {
3890
3891         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
3892
3893     } elsif ($deviceid eq 'keyboard') {
3894
3895         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
3896
3897     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3898
3899         die "usb hotplug currently not reliable\n";
3900         # since we can't reliably hot unplug all added usb devices and usb
3901         # passthrough breaks live migration we disable usb hotplugging for now
3902         #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3903
3904     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3905
3906         qemu_iothread_add($vmid, $deviceid, $device);
3907
3908         qemu_driveadd($storecfg, $vmid, $device);
3909         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3910
3911         qemu_deviceadd($vmid, $devicefull);
3912         eval { qemu_deviceaddverify($vmid, $deviceid); };
3913         if (my $err = $@) {
3914             eval { qemu_drivedel($vmid, $deviceid); };
3915             warn $@ if $@;
3916             die $err;
3917         }
3918
3919     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3920
3921
3922         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3923         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3924         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3925
3926         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3927
3928         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3929             qemu_iothread_add($vmid, $deviceid, $device);
3930             $devicefull .= ",iothread=iothread-$deviceid";
3931         }
3932
3933         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3934             $devicefull .= ",num_queues=$device->{queues}";
3935         }
3936
3937         qemu_deviceadd($vmid, $devicefull);
3938         qemu_deviceaddverify($vmid, $deviceid);
3939
3940     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3941
3942         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
3943         qemu_driveadd($storecfg, $vmid, $device);
3944
3945         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3946         eval { qemu_deviceadd($vmid, $devicefull); };
3947         if (my $err = $@) {
3948             eval { qemu_drivedel($vmid, $deviceid); };
3949             warn $@ if $@;
3950             die $err;
3951         }
3952
3953     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3954
3955         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
3956
3957         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
3958         my $use_old_bios_files = undef;
3959         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3960
3961         my $netdevicefull = print_netdevice_full(
3962             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
3963         qemu_deviceadd($vmid, $netdevicefull);
3964         eval {
3965             qemu_deviceaddverify($vmid, $deviceid);
3966             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
3967         };
3968         if (my $err = $@) {
3969             eval { qemu_netdevdel($vmid, $deviceid); };
3970             warn $@ if $@;
3971             die $err;
3972         }
3973
3974     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3975
3976         my $bridgeid = $2;
3977         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3978         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3979
3980         qemu_deviceadd($vmid, $devicefull);
3981         qemu_deviceaddverify($vmid, $deviceid);
3982
3983     } else {
3984         die "can't hotplug device '$deviceid'\n";
3985     }
3986
3987     return 1;
3988 }
3989
3990 # fixme: this should raise exceptions on error!
3991 sub vm_deviceunplug {
3992     my ($vmid, $conf, $deviceid) = @_;
3993
3994     my $devices_list = vm_devices_list($vmid);
3995     return 1 if !defined($devices_list->{$deviceid});
3996
3997     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
3998     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
3999
4000     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4001
4002         qemu_devicedel($vmid, $deviceid);
4003
4004     } elsif ($deviceid =~ m/^usb\d+$/) {
4005
4006         die "usb hotplug currently not reliable\n";
4007         # when unplugging usb devices this way, there may be remaining usb
4008         # controllers/hubs so we disable it for now
4009         #qemu_devicedel($vmid, $deviceid);
4010         #qemu_devicedelverify($vmid, $deviceid);
4011
4012     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4013
4014         qemu_devicedel($vmid, $deviceid);
4015         qemu_devicedelverify($vmid, $deviceid);
4016         qemu_drivedel($vmid, $deviceid);
4017         qemu_iothread_del($conf, $vmid, $deviceid);
4018
4019     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4020
4021         qemu_devicedel($vmid, $deviceid);
4022         qemu_devicedelverify($vmid, $deviceid);
4023         qemu_iothread_del($conf, $vmid, $deviceid);
4024
4025     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4026
4027         qemu_devicedel($vmid, $deviceid);
4028         qemu_drivedel($vmid, $deviceid);
4029         qemu_deletescsihw($conf, $vmid, $deviceid);
4030
4031     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4032
4033         qemu_devicedel($vmid, $deviceid);
4034         qemu_devicedelverify($vmid, $deviceid);
4035         qemu_netdevdel($vmid, $deviceid);
4036
4037     } else {
4038         die "can't unplug device '$deviceid'\n";
4039     }
4040
4041     return 1;
4042 }
4043
4044 sub qemu_deviceadd {
4045     my ($vmid, $devicefull) = @_;
4046
4047     $devicefull = "driver=".$devicefull;
4048     my %options =  split(/[=,]/, $devicefull);
4049
4050     mon_cmd($vmid, "device_add" , %options);
4051 }
4052
4053 sub qemu_devicedel {
4054     my ($vmid, $deviceid) = @_;
4055
4056     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4057 }
4058
4059 sub qemu_iothread_add {
4060     my($vmid, $deviceid, $device) = @_;
4061
4062     if ($device->{iothread}) {
4063         my $iothreads = vm_iothreads_list($vmid);
4064         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4065     }
4066 }
4067
4068 sub qemu_iothread_del {
4069     my($conf, $vmid, $deviceid) = @_;
4070
4071     my $confid = $deviceid;
4072     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4073         $confid = 'scsi' . $1;
4074     }
4075     my $device = parse_drive($confid, $conf->{$confid});
4076     if ($device->{iothread}) {
4077         my $iothreads = vm_iothreads_list($vmid);
4078         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4079     }
4080 }
4081
4082 sub qemu_objectadd {
4083     my($vmid, $objectid, $qomtype) = @_;
4084
4085     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4086
4087     return 1;
4088 }
4089
4090 sub qemu_objectdel {
4091     my($vmid, $objectid) = @_;
4092
4093     mon_cmd($vmid, "object-del", id => $objectid);
4094
4095     return 1;
4096 }
4097
4098 sub qemu_driveadd {
4099     my ($storecfg, $vmid, $device) = @_;
4100
4101     my $kvmver = get_running_qemu_version($vmid);
4102     my $io_uring = min_version($kvmver, 6, 0);
4103     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4104     $drive =~ s/\\/\\\\/g;
4105     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4106
4107     # If the command succeeds qemu prints: "OK"
4108     return 1 if $ret =~ m/OK/s;
4109
4110     die "adding drive failed: $ret\n";
4111 }
4112
4113 sub qemu_drivedel {
4114     my($vmid, $deviceid) = @_;
4115
4116     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4117     $ret =~ s/^\s+//;
4118
4119     return 1 if $ret eq "";
4120
4121     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4122     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4123
4124     die "deleting drive $deviceid failed : $ret\n";
4125 }
4126
4127 sub qemu_deviceaddverify {
4128     my ($vmid, $deviceid) = @_;
4129
4130     for (my $i = 0; $i <= 5; $i++) {
4131          my $devices_list = vm_devices_list($vmid);
4132          return 1 if defined($devices_list->{$deviceid});
4133          sleep 1;
4134     }
4135
4136     die "error on hotplug device '$deviceid'\n";
4137 }
4138
4139
4140 sub qemu_devicedelverify {
4141     my ($vmid, $deviceid) = @_;
4142
4143     # need to verify that the device is correctly removed as device_del
4144     # is async and empty return is not reliable
4145
4146     for (my $i = 0; $i <= 5; $i++) {
4147          my $devices_list = vm_devices_list($vmid);
4148          return 1 if !defined($devices_list->{$deviceid});
4149          sleep 1;
4150     }
4151
4152     die "error on hot-unplugging device '$deviceid'\n";
4153 }
4154
4155 sub qemu_findorcreatescsihw {
4156     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4157
4158     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4159
4160     my $scsihwid="$controller_prefix$controller";
4161     my $devices_list = vm_devices_list($vmid);
4162
4163     if(!defined($devices_list->{$scsihwid})) {
4164         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4165     }
4166
4167     return 1;
4168 }
4169
4170 sub qemu_deletescsihw {
4171     my ($conf, $vmid, $opt) = @_;
4172
4173     my $device = parse_drive($opt, $conf->{$opt});
4174
4175     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4176         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4177         return 1;
4178     }
4179
4180     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4181
4182     my $devices_list = vm_devices_list($vmid);
4183     foreach my $opt (keys %{$devices_list}) {
4184         if (is_valid_drivename($opt)) {
4185             my $drive = parse_drive($opt, $conf->{$opt});
4186             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4187                 return 1;
4188             }
4189         }
4190     }
4191
4192     my $scsihwid="scsihw$controller";
4193
4194     vm_deviceunplug($vmid, $conf, $scsihwid);
4195
4196     return 1;
4197 }
4198
4199 sub qemu_add_pci_bridge {
4200     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4201
4202     my $bridges = {};
4203
4204     my $bridgeid;
4205
4206     print_pci_addr($device, $bridges, $arch, $machine_type);
4207
4208     while (my ($k, $v) = each %$bridges) {
4209         $bridgeid = $k;
4210     }
4211     return 1 if !defined($bridgeid) || $bridgeid < 1;
4212
4213     my $bridge = "pci.$bridgeid";
4214     my $devices_list = vm_devices_list($vmid);
4215
4216     if (!defined($devices_list->{$bridge})) {
4217         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4218     }
4219
4220     return 1;
4221 }
4222
4223 sub qemu_set_link_status {
4224     my ($vmid, $device, $up) = @_;
4225
4226     mon_cmd($vmid, "set_link", name => $device,
4227                up => $up ? JSON::true : JSON::false);
4228 }
4229
4230 sub qemu_netdevadd {
4231     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4232
4233     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4234     my %options =  split(/[=,]/, $netdev);
4235
4236     if (defined(my $vhost = $options{vhost})) {
4237         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4238     }
4239
4240     if (defined(my $queues = $options{queues})) {
4241         $options{queues} = $queues + 0;
4242     }
4243
4244     mon_cmd($vmid, "netdev_add",  %options);
4245     return 1;
4246 }
4247
4248 sub qemu_netdevdel {
4249     my ($vmid, $deviceid) = @_;
4250
4251     mon_cmd($vmid, "netdev_del", id => $deviceid);
4252 }
4253
4254 sub qemu_usb_hotplug {
4255     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4256
4257     return if !$device;
4258
4259     # remove the old one first
4260     vm_deviceunplug($vmid, $conf, $deviceid);
4261
4262     # check if xhci controller is necessary and available
4263     if ($device->{usb3}) {
4264
4265         my $devicelist = vm_devices_list($vmid);
4266
4267         if (!$devicelist->{xhci}) {
4268             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4269             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4270         }
4271     }
4272     my $d = parse_usb_device($device->{host});
4273     $d->{usb3} = $device->{usb3};
4274
4275     # add the new one
4276     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4277 }
4278
4279 sub qemu_cpu_hotplug {
4280     my ($vmid, $conf, $vcpus) = @_;
4281
4282     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4283
4284     my $sockets = 1;
4285     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4286     $sockets = $conf->{sockets} if  $conf->{sockets};
4287     my $cores = $conf->{cores} || 1;
4288     my $maxcpus = $sockets * $cores;
4289
4290     $vcpus = $maxcpus if !$vcpus;
4291
4292     die "you can't add more vcpus than maxcpus\n"
4293         if $vcpus > $maxcpus;
4294
4295     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4296
4297     if ($vcpus < $currentvcpus) {
4298
4299         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4300
4301             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4302                 qemu_devicedel($vmid, "cpu$i");
4303                 my $retry = 0;
4304                 my $currentrunningvcpus = undef;
4305                 while (1) {
4306                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4307                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4308                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4309                     $retry++;
4310                     sleep 1;
4311                 }
4312                 #update conf after each succesfull cpu unplug
4313                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4314                 PVE::QemuConfig->write_config($vmid, $conf);
4315             }
4316         } else {
4317             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4318         }
4319
4320         return;
4321     }
4322
4323     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4324     die "vcpus in running vm does not match its configuration\n"
4325         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4326
4327     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4328
4329         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4330             my $cpustr = print_cpu_device($conf, $i);
4331             qemu_deviceadd($vmid, $cpustr);
4332
4333             my $retry = 0;
4334             my $currentrunningvcpus = undef;
4335             while (1) {
4336                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4337                 last if scalar(@{$currentrunningvcpus}) == $i;
4338                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4339                 sleep 1;
4340                 $retry++;
4341             }
4342             #update conf after each succesfull cpu hotplug
4343             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4344             PVE::QemuConfig->write_config($vmid, $conf);
4345         }
4346     } else {
4347
4348         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4349             mon_cmd($vmid, "cpu-add", id => int($i));
4350         }
4351     }
4352 }
4353
4354 sub qemu_block_set_io_throttle {
4355     my ($vmid, $deviceid,
4356         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4357         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4358         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4359         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4360
4361     return if !check_running($vmid) ;
4362
4363     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4364         bps => int($bps),
4365         bps_rd => int($bps_rd),
4366         bps_wr => int($bps_wr),
4367         iops => int($iops),
4368         iops_rd => int($iops_rd),
4369         iops_wr => int($iops_wr),
4370         bps_max => int($bps_max),
4371         bps_rd_max => int($bps_rd_max),
4372         bps_wr_max => int($bps_wr_max),
4373         iops_max => int($iops_max),
4374         iops_rd_max => int($iops_rd_max),
4375         iops_wr_max => int($iops_wr_max),
4376         bps_max_length => int($bps_max_length),
4377         bps_rd_max_length => int($bps_rd_max_length),
4378         bps_wr_max_length => int($bps_wr_max_length),
4379         iops_max_length => int($iops_max_length),
4380         iops_rd_max_length => int($iops_rd_max_length),
4381         iops_wr_max_length => int($iops_wr_max_length),
4382     );
4383
4384 }
4385
4386 sub qemu_block_resize {
4387     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4388
4389     my $running = check_running($vmid);
4390
4391     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4392
4393     return if !$running;
4394
4395     my $padding = (1024 - $size % 1024) % 1024;
4396     $size = $size + $padding;
4397
4398     mon_cmd(
4399         $vmid,
4400         "block_resize",
4401         device => $deviceid,
4402         size => int($size),
4403         timeout => 60,
4404     );
4405 }
4406
4407 sub qemu_volume_snapshot {
4408     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4409
4410     my $running = check_running($vmid);
4411
4412     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4413         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4414     } else {
4415         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4416     }
4417 }
4418
4419 sub qemu_volume_snapshot_delete {
4420     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4421
4422     my $running = check_running($vmid);
4423
4424     if($running) {
4425
4426         $running = undef;
4427         my $conf = PVE::QemuConfig->load_config($vmid);
4428         PVE::QemuConfig->foreach_volume($conf, sub {
4429             my ($ds, $drive) = @_;
4430             $running = 1 if $drive->{file} eq $volid;
4431         });
4432     }
4433
4434     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4435         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4436     } else {
4437         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4438     }
4439 }
4440
4441 sub set_migration_caps {
4442     my ($vmid, $savevm) = @_;
4443
4444     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4445
4446     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4447     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4448
4449     my $cap_ref = [];
4450
4451     my $enabled_cap = {
4452         "auto-converge" => 1,
4453         "xbzrle" => 1,
4454         "x-rdma-pin-all" => 0,
4455         "zero-blocks" => 0,
4456         "compress" => 0,
4457         "dirty-bitmaps" => $dirty_bitmaps,
4458     };
4459
4460     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4461
4462     for my $supported_capability (@$supported_capabilities) {
4463         push @$cap_ref, {
4464             capability => $supported_capability->{capability},
4465             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4466         };
4467     }
4468
4469     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4470 }
4471
4472 sub foreach_volid {
4473     my ($conf, $func, @param) = @_;
4474
4475     my $volhash = {};
4476
4477     my $test_volid = sub {
4478         my ($key, $drive, $snapname) = @_;
4479
4480         my $volid = $drive->{file};
4481         return if !$volid;
4482
4483         $volhash->{$volid}->{cdrom} //= 1;
4484         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4485
4486         my $replicate = $drive->{replicate} // 1;
4487         $volhash->{$volid}->{replicate} //= 0;
4488         $volhash->{$volid}->{replicate} = 1 if $replicate;
4489
4490         $volhash->{$volid}->{shared} //= 0;
4491         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4492
4493         $volhash->{$volid}->{referenced_in_config} //= 0;
4494         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4495
4496         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4497             if defined($snapname);
4498
4499         my $size = $drive->{size};
4500         $volhash->{$volid}->{size} //= $size if $size;
4501
4502         $volhash->{$volid}->{is_vmstate} //= 0;
4503         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4504
4505         $volhash->{$volid}->{is_unused} //= 0;
4506         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4507
4508         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4509     };
4510
4511     my $include_opts = {
4512         extra_keys => ['vmstate'],
4513         include_unused => 1,
4514     };
4515
4516     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4517     foreach my $snapname (keys %{$conf->{snapshots}}) {
4518         my $snap = $conf->{snapshots}->{$snapname};
4519         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4520     }
4521
4522     foreach my $volid (keys %$volhash) {
4523         &$func($volid, $volhash->{$volid}, @param);
4524     }
4525 }
4526
4527 my $fast_plug_option = {
4528     'lock' => 1,
4529     'name' => 1,
4530     'onboot' => 1,
4531     'shares' => 1,
4532     'startup' => 1,
4533     'description' => 1,
4534     'protection' => 1,
4535     'vmstatestorage' => 1,
4536     'hookscript' => 1,
4537     'tags' => 1,
4538 };
4539
4540 # hotplug changes in [PENDING]
4541 # $selection hash can be used to only apply specified options, for
4542 # example: { cores => 1 } (only apply changed 'cores')
4543 # $errors ref is used to return error messages
4544 sub vmconfig_hotplug_pending {
4545     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4546
4547     my $defaults = load_defaults();
4548     my $arch = get_vm_arch($conf);
4549     my $machine_type = get_vm_machine($conf, undef, $arch);
4550
4551     # commit values which do not have any impact on running VM first
4552     # Note: those option cannot raise errors, we we do not care about
4553     # $selection and always apply them.
4554
4555     my $add_error = sub {
4556         my ($opt, $msg) = @_;
4557         $errors->{$opt} = "hotplug problem - $msg";
4558     };
4559
4560     my $changes = 0;
4561     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4562         if ($fast_plug_option->{$opt}) {
4563             $conf->{$opt} = $conf->{pending}->{$opt};
4564             delete $conf->{pending}->{$opt};
4565             $changes = 1;
4566         }
4567     }
4568
4569     if ($changes) {
4570         PVE::QemuConfig->write_config($vmid, $conf);
4571     }
4572
4573     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4574
4575     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4576     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4577     foreach my $opt (sort keys %$pending_delete_hash) {
4578         next if $selection && !$selection->{$opt};
4579         my $force = $pending_delete_hash->{$opt}->{force};
4580         eval {
4581             if ($opt eq 'hotplug') {
4582                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4583             } elsif ($opt eq 'tablet') {
4584                 die "skip\n" if !$hotplug_features->{usb};
4585                 if ($defaults->{tablet}) {
4586                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4587                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4588                         if $arch eq 'aarch64';
4589                 } else {
4590                     vm_deviceunplug($vmid, $conf, 'tablet');
4591                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4592                 }
4593             } elsif ($opt =~ m/^usb\d+/) {
4594                 die "skip\n";
4595                 # since we cannot reliably hot unplug usb devices we are disabling it
4596                 #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4597                 #vm_deviceunplug($vmid, $conf, $opt);
4598             } elsif ($opt eq 'vcpus') {
4599                 die "skip\n" if !$hotplug_features->{cpu};
4600                 qemu_cpu_hotplug($vmid, $conf, undef);
4601             } elsif ($opt eq 'balloon') {
4602                 # enable balloon device is not hotpluggable
4603                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4604                 # here we reset the ballooning value to memory
4605                 my $balloon = $conf->{memory} || $defaults->{memory};
4606                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4607             } elsif ($fast_plug_option->{$opt}) {
4608                 # do nothing
4609             } elsif ($opt =~ m/^net(\d+)$/) {
4610                 die "skip\n" if !$hotplug_features->{network};
4611                 vm_deviceunplug($vmid, $conf, $opt);
4612             } elsif (is_valid_drivename($opt)) {
4613                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4614                 vm_deviceunplug($vmid, $conf, $opt);
4615                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4616             } elsif ($opt =~ m/^memory$/) {
4617                 die "skip\n" if !$hotplug_features->{memory};
4618                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4619             } elsif ($opt eq 'cpuunits') {
4620                 $cgroup->change_cpu_shares(undef, $defaults->{cpuunits});
4621             } elsif ($opt eq 'cpulimit') {
4622                 $cgroup->change_cpu_quota(-1, 100000);
4623             } else {
4624                 die "skip\n";
4625             }
4626         };
4627         if (my $err = $@) {
4628             &$add_error($opt, $err) if $err ne "skip\n";
4629         } else {
4630             delete $conf->{$opt};
4631             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4632         }
4633     }
4634
4635     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4636     $apply_pending_cloudinit = sub {
4637         return if $apply_pending_cloudinit_done; # once is enough
4638         $apply_pending_cloudinit_done = 1; # once is enough
4639
4640         my ($key, $value) = @_;
4641
4642         my @cloudinit_opts = keys %$confdesc_cloudinit;
4643         foreach my $opt (keys %{$conf->{pending}}) {
4644             next if !grep { $_ eq $opt } @cloudinit_opts;
4645             $conf->{$opt} = delete $conf->{pending}->{$opt};
4646         }
4647
4648         my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4649         foreach my $opt (sort keys %$pending_delete_hash) {
4650             next if !grep { $_ eq $opt } @cloudinit_opts;
4651             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4652             delete $conf->{$opt};
4653         }
4654
4655         my $new_conf = { %$conf };
4656         $new_conf->{$key} = $value;
4657         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4658     };
4659
4660     foreach my $opt (keys %{$conf->{pending}}) {
4661         next if $selection && !$selection->{$opt};
4662         my $value = $conf->{pending}->{$opt};
4663         eval {
4664             if ($opt eq 'hotplug') {
4665                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4666             } elsif ($opt eq 'tablet') {
4667                 die "skip\n" if !$hotplug_features->{usb};
4668                 if ($value == 1) {
4669                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4670                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4671                         if $arch eq 'aarch64';
4672                 } elsif ($value == 0) {
4673                     vm_deviceunplug($vmid, $conf, 'tablet');
4674                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4675                 }
4676             } elsif ($opt =~ m/^usb\d+$/) {
4677                 die "skip\n";
4678                 # since we cannot reliably hot unplug usb devices we disable it for now
4679                 #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4680                 #my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4681                 #die "skip\n" if !$d;
4682                 #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4683             } elsif ($opt eq 'vcpus') {
4684                 die "skip\n" if !$hotplug_features->{cpu};
4685                 qemu_cpu_hotplug($vmid, $conf, $value);
4686             } elsif ($opt eq 'balloon') {
4687                 # enable/disable balloning device is not hotpluggable
4688                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4689                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4690                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4691
4692                 # allow manual ballooning if shares is set to zero
4693                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4694                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4695                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4696                 }
4697             } elsif ($opt =~ m/^net(\d+)$/) {
4698                 # some changes can be done without hotplug
4699                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4700                                     $vmid, $opt, $value, $arch, $machine_type);
4701             } elsif (is_valid_drivename($opt)) {
4702                 die "skip\n" if $opt eq 'efidisk0';
4703                 # some changes can be done without hotplug
4704                 my $drive = parse_drive($opt, $value);
4705                 if (drive_is_cloudinit($drive)) {
4706                     &$apply_pending_cloudinit($opt, $value);
4707                 }
4708                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4709                                      $vmid, $opt, $value, $arch, $machine_type);
4710             } elsif ($opt =~ m/^memory$/) { #dimms
4711                 die "skip\n" if !$hotplug_features->{memory};
4712                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4713             } elsif ($opt eq 'cpuunits') {
4714                 $cgroup->change_cpu_shares($conf->{pending}->{$opt}, $defaults->{cpuunits});
4715             } elsif ($opt eq 'cpulimit') {
4716                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4717                 $cgroup->change_cpu_quota($cpulimit, 100000);
4718             } else {
4719                 die "skip\n";  # skip non-hot-pluggable options
4720             }
4721         };
4722         if (my $err = $@) {
4723             &$add_error($opt, $err) if $err ne "skip\n";
4724         } else {
4725             $conf->{$opt} = $value;
4726             delete $conf->{pending}->{$opt};
4727         }
4728     }
4729
4730     PVE::QemuConfig->write_config($vmid, $conf);
4731 }
4732
4733 sub try_deallocate_drive {
4734     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4735
4736     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4737         my $volid = $drive->{file};
4738         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4739             my $sid = PVE::Storage::parse_volume_id($volid);
4740             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4741
4742             # check if the disk is really unused
4743             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4744                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4745             PVE::Storage::vdisk_free($storecfg, $volid);
4746             return 1;
4747         } else {
4748             # If vm is not owner of this disk remove from config
4749             return 1;
4750         }
4751     }
4752
4753     return;
4754 }
4755
4756 sub vmconfig_delete_or_detach_drive {
4757     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4758
4759     my $drive = parse_drive($opt, $conf->{$opt});
4760
4761     my $rpcenv = PVE::RPCEnvironment::get();
4762     my $authuser = $rpcenv->get_user();
4763
4764     if ($force) {
4765         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4766         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4767     } else {
4768         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4769     }
4770 }
4771
4772
4773
4774 sub vmconfig_apply_pending {
4775     my ($vmid, $conf, $storecfg, $errors) = @_;
4776
4777     my $add_apply_error = sub {
4778         my ($opt, $msg) = @_;
4779         my $err_msg = "unable to apply pending change $opt : $msg";
4780         $errors->{$opt} = $err_msg;
4781         warn $err_msg;
4782     };
4783
4784     # cold plug
4785
4786     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4787     foreach my $opt (sort keys %$pending_delete_hash) {
4788         my $force = $pending_delete_hash->{$opt}->{force};
4789         eval {
4790             if ($opt =~ m/^unused/) {
4791                 die "internal error";
4792             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4793                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4794             }
4795         };
4796         if (my $err = $@) {
4797             $add_apply_error->($opt, $err);
4798         } else {
4799             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4800             delete $conf->{$opt};
4801         }
4802     }
4803
4804     PVE::QemuConfig->cleanup_pending($conf);
4805
4806     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4807         next if $opt eq 'delete'; # just to be sure
4808         eval {
4809             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4810                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4811             }
4812         };
4813         if (my $err = $@) {
4814             $add_apply_error->($opt, $err);
4815         } else {
4816             $conf->{$opt} = delete $conf->{pending}->{$opt};
4817         }
4818     }
4819
4820     # write all changes at once to avoid unnecessary i/o
4821     PVE::QemuConfig->write_config($vmid, $conf);
4822 }
4823
4824 sub vmconfig_update_net {
4825     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4826
4827     my $newnet = parse_net($value);
4828
4829     if ($conf->{$opt}) {
4830         my $oldnet = parse_net($conf->{$opt});
4831
4832         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4833             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4834             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4835             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4836
4837             # for non online change, we try to hot-unplug
4838             die "skip\n" if !$hotplug;
4839             vm_deviceunplug($vmid, $conf, $opt);
4840         } else {
4841
4842             die "internal error" if $opt !~ m/net(\d+)/;
4843             my $iface = "tap${vmid}i$1";
4844
4845             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4846                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4847                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4848                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4849                 PVE::Network::tap_unplug($iface);
4850
4851                 if ($have_sdn) {
4852                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4853                 } else {
4854                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4855                 }
4856             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4857                 # Rate can be applied on its own but any change above needs to
4858                 # include the rate in tap_plug since OVS resets everything.
4859                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4860             }
4861
4862             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4863                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4864             }
4865
4866             return 1;
4867         }
4868     }
4869
4870     if ($hotplug) {
4871         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4872     } else {
4873         die "skip\n";
4874     }
4875 }
4876
4877 sub vmconfig_update_disk {
4878     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4879
4880     my $drive = parse_drive($opt, $value);
4881
4882     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
4883         my $media = $drive->{media} || 'disk';
4884         my $oldmedia = $old_drive->{media} || 'disk';
4885         die "unable to change media type\n" if $media ne $oldmedia;
4886
4887         if (!drive_is_cdrom($old_drive)) {
4888
4889             if ($drive->{file} ne $old_drive->{file}) {
4890
4891                 die "skip\n" if !$hotplug;
4892
4893                 # unplug and register as unused
4894                 vm_deviceunplug($vmid, $conf, $opt);
4895                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4896
4897             } else {
4898                 # update existing disk
4899
4900                 # skip non hotpluggable value
4901                 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4902                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4903                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4904                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
4905                     safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
4906                     die "skip\n";
4907                 }
4908
4909                 # apply throttle
4910                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4911                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4912                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4913                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4914                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4915                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4916                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4917                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4918                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4919                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4920                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4921                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4922                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4923                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4924                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4925                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4926                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4927                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4928
4929                     qemu_block_set_io_throttle(
4930                         $vmid,"drive-$opt",
4931                         ($drive->{mbps} || 0)*1024*1024,
4932                         ($drive->{mbps_rd} || 0)*1024*1024,
4933                         ($drive->{mbps_wr} || 0)*1024*1024,
4934                          $drive->{iops} || 0,
4935                          $drive->{iops_rd} || 0,
4936                          $drive->{iops_wr} || 0,
4937                         ($drive->{mbps_max} || 0)*1024*1024,
4938                         ($drive->{mbps_rd_max} || 0)*1024*1024,
4939                         ($drive->{mbps_wr_max} || 0)*1024*1024,
4940                          $drive->{iops_max} || 0,
4941                          $drive->{iops_rd_max} || 0,
4942                          $drive->{iops_wr_max} || 0,
4943                          $drive->{bps_max_length} || 1,
4944                          $drive->{bps_rd_max_length} || 1,
4945                          $drive->{bps_wr_max_length} || 1,
4946                          $drive->{iops_max_length} || 1,
4947                          $drive->{iops_rd_max_length} || 1,
4948                          $drive->{iops_wr_max_length} || 1,
4949                     );
4950
4951                 }
4952
4953                 return 1;
4954             }
4955
4956         } else { # cdrom
4957
4958             if ($drive->{file} eq 'none') {
4959                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4960                 if (drive_is_cloudinit($old_drive)) {
4961                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4962                 }
4963             } else {
4964                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4965
4966                 # force eject if locked
4967                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4968
4969                 if ($path) {
4970                     mon_cmd($vmid, "blockdev-change-medium",
4971                         id => "$opt", filename => "$path");
4972                 }
4973             }
4974
4975             return 1;
4976         }
4977     }
4978
4979     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4980     # hotplug new disks
4981     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4982     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
4983 }
4984
4985 # called in locked context by incoming migration
4986 sub vm_migrate_get_nbd_disks {
4987     my ($storecfg, $conf, $replicated_volumes) = @_;
4988
4989     my $local_volumes = {};
4990     PVE::QemuConfig->foreach_volume($conf, sub {
4991         my ($ds, $drive) = @_;
4992
4993         return if drive_is_cdrom($drive);
4994
4995         my $volid = $drive->{file};
4996
4997         return if !$volid;
4998
4999         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5000
5001         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5002         return if $scfg->{shared};
5003
5004         # replicated disks re-use existing state via bitmap
5005         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5006         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
5007     });
5008     return $local_volumes;
5009 }
5010
5011 # called in locked context by incoming migration
5012 sub vm_migrate_alloc_nbd_disks {
5013     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5014
5015     my $format = undef;
5016
5017     my $nbd = {};
5018     foreach my $opt (sort keys %$source_volumes) {
5019         my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
5020
5021         if ($use_existing) {
5022             $nbd->{$opt}->{drivestr} = print_drive($drive);
5023             $nbd->{$opt}->{volid} = $volid;
5024             $nbd->{$opt}->{replicated} = 1;
5025             next;
5026         }
5027
5028         # If a remote storage is specified and the format of the original
5029         # volume is not available there, fall back to the default format.
5030         # Otherwise use the same format as the original.
5031         if (!$storagemap->{identity}) {
5032             $storeid = map_storage($storagemap, $storeid);
5033             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5034             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5035             my $fileFormat = qemu_img_format($scfg, $volname);
5036             $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
5037         } else {
5038             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5039             $format = qemu_img_format($scfg, $volname);
5040         }
5041
5042         my $size = $drive->{size} / 1024;
5043         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5044         my $newdrive = $drive;
5045         $newdrive->{format} = $format;
5046         $newdrive->{file} = $newvolid;
5047         my $drivestr = print_drive($newdrive);
5048         $nbd->{$opt}->{drivestr} = $drivestr;
5049         $nbd->{$opt}->{volid} = $newvolid;
5050     }
5051
5052     return $nbd;
5053 }
5054
5055 # see vm_start_nolock for parameters, additionally:
5056 # migrate_opts:
5057 #   storagemap = parsed storage map for allocating NBD disks
5058 sub vm_start {
5059     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5060
5061     return PVE::QemuConfig->lock_config($vmid, sub {
5062         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5063
5064         die "you can't start a vm if it's a template\n"
5065             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5066
5067         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5068         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5069
5070         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5071
5072         if ($has_backup_lock && $running) {
5073             # a backup is currently running, attempt to start the guest in the
5074             # existing QEMU instance
5075             return vm_resume($vmid);
5076         }
5077
5078         PVE::QemuConfig->check_lock($conf)
5079             if !($params->{skiplock} || $has_suspended_lock);
5080
5081         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5082
5083         die "VM $vmid already running\n" if $running;
5084
5085         if (my $storagemap = $migrate_opts->{storagemap}) {
5086             my $replicated = $migrate_opts->{replicated_volumes};
5087             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5088             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5089
5090             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5091                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5092             }
5093         }
5094
5095         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5096     });
5097 }
5098
5099
5100 # params:
5101 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5102 #   skiplock => 0/1, skip checking for config lock
5103 #   skiptemplate => 0/1, skip checking whether VM is template
5104 #   forcemachine => to force Qemu machine (rollback/migration)
5105 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5106 #   timeout => in seconds
5107 #   paused => start VM in paused state (backup)
5108 #   resume => resume from hibernation
5109 #   pbs-backing => {
5110 #      sata0 => {
5111 #         repository
5112 #         snapshot
5113 #         keyfile
5114 #         archive
5115 #      },
5116 #      virtio2 => ...
5117 #   }
5118 # migrate_opts:
5119 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5120 #   migratedfrom => source node
5121 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5122 #   network => CIDR of migration network
5123 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5124 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5125 #   replicated_volumes = which volids should be re-used with bitmaps for nbd migration
5126 sub vm_start_nolock {
5127     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5128
5129     my $statefile = $params->{statefile};
5130     my $resume = $params->{resume};
5131
5132     my $migratedfrom = $migrate_opts->{migratedfrom};
5133     my $migration_type = $migrate_opts->{type};
5134
5135     my $res = {};
5136
5137     # clean up leftover reboot request files
5138     eval { clear_reboot_request($vmid); };
5139     warn $@ if $@;
5140
5141     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5142         vmconfig_apply_pending($vmid, $conf, $storecfg);
5143         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5144     }
5145
5146     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5147
5148     my $defaults = load_defaults();
5149
5150     # set environment variable useful inside network script
5151     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5152
5153     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5154
5155     my $forcemachine = $params->{forcemachine};
5156     my $forcecpu = $params->{forcecpu};
5157     if ($resume) {
5158         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5159         $forcemachine = $conf->{runningmachine};
5160         $forcecpu = $conf->{runningcpu};
5161         print "Resuming suspended VM\n";
5162     }
5163
5164     my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid,
5165         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5166
5167     my $migration_ip;
5168     my $get_migration_ip = sub {
5169         my ($nodename) = @_;
5170
5171         return $migration_ip if defined($migration_ip);
5172
5173         my $cidr = $migrate_opts->{network};
5174
5175         if (!defined($cidr)) {
5176             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5177             $cidr = $dc_conf->{migration}->{network};
5178         }
5179
5180         if (defined($cidr)) {
5181             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5182
5183             die "could not get IP: no address configured on local " .
5184                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5185
5186             die "could not get IP: multiple addresses configured on local " .
5187                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5188
5189             $migration_ip = @$ips[0];
5190         }
5191
5192         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5193             if !defined($migration_ip);
5194
5195         return $migration_ip;
5196     };
5197
5198     my $migrate_uri;
5199     if ($statefile) {
5200         if ($statefile eq 'tcp') {
5201             my $localip = "localhost";
5202             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5203             my $nodename = nodename();
5204
5205             if (!defined($migration_type)) {
5206                 if (defined($datacenterconf->{migration}->{type})) {
5207                     $migration_type = $datacenterconf->{migration}->{type};
5208                 } else {
5209                     $migration_type = 'secure';
5210                 }
5211             }
5212
5213             if ($migration_type eq 'insecure') {
5214                 $localip = $get_migration_ip->($nodename);
5215                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5216             }
5217
5218             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5219             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5220             $migrate_uri = "tcp:${localip}:${migrate_port}";
5221             push @$cmd, '-incoming', $migrate_uri;
5222             push @$cmd, '-S';
5223
5224         } elsif ($statefile eq 'unix') {
5225             # should be default for secure migrations as a ssh TCP forward
5226             # tunnel is not deterministic reliable ready and fails regurarly
5227             # to set up in time, so use UNIX socket forwards
5228             my $socket_addr = "/run/qemu-server/$vmid.migrate";
5229             unlink $socket_addr;
5230
5231             $migrate_uri = "unix:$socket_addr";
5232
5233             push @$cmd, '-incoming', $migrate_uri;
5234             push @$cmd, '-S';
5235
5236         } elsif (-e $statefile) {
5237             push @$cmd, '-loadstate', $statefile;
5238         } else {
5239             my $statepath = PVE::Storage::path($storecfg, $statefile);
5240             push @$vollist, $statefile;
5241             push @$cmd, '-loadstate', $statepath;
5242         }
5243     } elsif ($params->{paused}) {
5244         push @$cmd, '-S';
5245     }
5246
5247     # host pci devices
5248     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5249       my $d = parse_hostpci($conf->{"hostpci$i"});
5250       next if !$d;
5251       my $pcidevices = $d->{pciid};
5252       foreach my $pcidevice (@$pcidevices) {
5253             my $pciid = $pcidevice->{id};
5254
5255             my $info = PVE::SysFSTools::pci_device_info("$pciid");
5256             die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5257             die "no pci device info for device '$pciid'\n" if !$info;
5258
5259             if ($d->{mdev}) {
5260                 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5261                 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5262             } else {
5263                 die "can't unbind/bind PCI group to VFIO '$pciid'\n"
5264                     if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5265                 die "can't reset PCI device '$pciid'\n"
5266                     if $info->{has_fl_reset} && !PVE::SysFSTools::pci_dev_reset($info);
5267             }
5268       }
5269     }
5270
5271     PVE::Storage::activate_volumes($storecfg, $vollist);
5272
5273     eval {
5274         run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5275             outfunc => sub {}, errfunc => sub {});
5276     };
5277     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5278     # timeout should be more than enough here...
5279     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5280
5281     my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5282                                               : $defaults->{cpuunits};
5283
5284     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5285     my %run_params = (
5286         timeout => $statefile ? undef : $start_timeout,
5287         umask => 0077,
5288         noerr => 1,
5289     );
5290
5291     # when migrating, prefix QEMU output so other side can pick up any
5292     # errors that might occur and show the user
5293     if ($migratedfrom) {
5294         $run_params{quiet} = 1;
5295         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5296     }
5297
5298     my %properties = (
5299         Slice => 'qemu.slice',
5300         KillMode => 'process',
5301         SendSIGKILL => 0,
5302         TimeoutStopUSec => ULONG_MAX, # infinity
5303     );
5304
5305     if (PVE::CGroup::cgroup_mode() == 2) {
5306         $properties{CPUWeight} = $cpuunits;
5307     } else {
5308         $properties{CPUShares} = $cpuunits;
5309     }
5310
5311     if (my $cpulimit = $conf->{cpulimit}) {
5312         $properties{CPUQuota} = int($cpulimit * 100);
5313     }
5314     $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5315
5316     my $run_qemu = sub {
5317         PVE::Tools::run_fork sub {
5318             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5319
5320             my $exitcode = run_command($cmd, %run_params);
5321             die "QEMU exited with code $exitcode\n" if $exitcode;
5322         };
5323     };
5324
5325     if ($conf->{hugepages}) {
5326
5327         my $code = sub {
5328             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5329             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5330
5331             PVE::QemuServer::Memory::hugepages_mount();
5332             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5333
5334             eval { $run_qemu->() };
5335             if (my $err = $@) {
5336                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5337                     if !$conf->{keephugepages};
5338                 die $err;
5339             }
5340
5341             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5342                 if !$conf->{keephugepages};
5343         };
5344         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5345
5346     } else {
5347         eval { $run_qemu->() };
5348     }
5349
5350     if (my $err = $@) {
5351         # deactivate volumes if start fails
5352         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5353         die "start failed: $err";
5354     }
5355
5356     print "migration listens on $migrate_uri\n" if $migrate_uri;
5357     $res->{migrate_uri} = $migrate_uri;
5358
5359     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5360         eval { mon_cmd($vmid, "cont"); };
5361         warn $@ if $@;
5362     }
5363
5364     #start nbd server for storage migration
5365     if (my $nbd = $migrate_opts->{nbd}) {
5366         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5367
5368         my $migrate_storage_uri;
5369         # nbd_protocol_version > 0 for unix socket support
5370         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5371             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5372             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5373             $migrate_storage_uri = "nbd:unix:$socket_path";
5374         } else {
5375             my $nodename = nodename();
5376             my $localip = $get_migration_ip->($nodename);
5377             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5378             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5379
5380             mon_cmd($vmid, "nbd-server-start", addr => {
5381                 type => 'inet',
5382                 data => {
5383                     host => "${localip}",
5384                     port => "${storage_migrate_port}",
5385                 },
5386             });
5387             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5388             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5389         }
5390
5391         $res->{migrate_storage_uri} = $migrate_storage_uri;
5392
5393         foreach my $opt (sort keys %$nbd) {
5394             my $drivestr = $nbd->{$opt}->{drivestr};
5395             my $volid = $nbd->{$opt}->{volid};
5396             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5397             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5398             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5399             print "re-using replicated volume: $opt - $volid\n"
5400                 if $nbd->{$opt}->{replicated};
5401
5402             $res->{drives}->{$opt} = $nbd->{$opt};
5403             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5404         }
5405     }
5406
5407     if ($migratedfrom) {
5408         eval {
5409             set_migration_caps($vmid);
5410         };
5411         warn $@ if $@;
5412
5413         if ($spice_port) {
5414             print "spice listens on port $spice_port\n";
5415             $res->{spice_port} = $spice_port;
5416             if ($migrate_opts->{spice_ticket}) {
5417                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5418                     $migrate_opts->{spice_ticket});
5419                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5420             }
5421         }
5422
5423     } else {
5424         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5425             if !$statefile && $conf->{balloon};
5426
5427         foreach my $opt (keys %$conf) {
5428             next if $opt !~  m/^net\d+$/;
5429             my $nicconf = parse_net($conf->{$opt});
5430             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5431         }
5432     }
5433
5434     mon_cmd($vmid, 'qom-set',
5435                 path => "machine/peripheral/balloon0",
5436                 property => "guest-stats-polling-interval",
5437                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5438
5439     if ($resume) {
5440         print "Resumed VM, removing state\n";
5441         if (my $vmstate = $conf->{vmstate}) {
5442             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5443             PVE::Storage::vdisk_free($storecfg, $vmstate);
5444         }
5445         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5446         PVE::QemuConfig->write_config($vmid, $conf);
5447     }
5448
5449     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5450
5451     return $res;
5452 }
5453
5454 sub vm_commandline {
5455     my ($storecfg, $vmid, $snapname) = @_;
5456
5457     my $conf = PVE::QemuConfig->load_config($vmid);
5458     my $forcemachine;
5459     my $forcecpu;
5460
5461     if ($snapname) {
5462         my $snapshot = $conf->{snapshots}->{$snapname};
5463         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5464
5465         # check for machine or CPU overrides in snapshot
5466         $forcemachine = $snapshot->{runningmachine};
5467         $forcecpu = $snapshot->{runningcpu};
5468
5469         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5470
5471         $conf = $snapshot;
5472     }
5473
5474     my $defaults = load_defaults();
5475
5476     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults,
5477         $forcemachine, $forcecpu);
5478
5479     return PVE::Tools::cmd2string($cmd);
5480 }
5481
5482 sub vm_reset {
5483     my ($vmid, $skiplock) = @_;
5484
5485     PVE::QemuConfig->lock_config($vmid, sub {
5486
5487         my $conf = PVE::QemuConfig->load_config($vmid);
5488
5489         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5490
5491         mon_cmd($vmid, "system_reset");
5492     });
5493 }
5494
5495 sub get_vm_volumes {
5496     my ($conf) = @_;
5497
5498     my $vollist = [];
5499     foreach_volid($conf, sub {
5500         my ($volid, $attr) = @_;
5501
5502         return if $volid =~ m|^/|;
5503
5504         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5505         return if !$sid;
5506
5507         push @$vollist, $volid;
5508     });
5509
5510     return $vollist;
5511 }
5512
5513 sub vm_stop_cleanup {
5514     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5515
5516     eval {
5517
5518         if (!$keepActive) {
5519             my $vollist = get_vm_volumes($conf);
5520             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5521         }
5522
5523         foreach my $ext (qw(mon qmp pid vnc qga)) {
5524             unlink "/var/run/qemu-server/${vmid}.$ext";
5525         }
5526
5527         if ($conf->{ivshmem}) {
5528             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5529             # just delete it for now, VMs which have this already open do not
5530             # are affected, but new VMs will get a separated one. If this
5531             # becomes an issue we either add some sort of ref-counting or just
5532             # add a "don't delete on stop" flag to the ivshmem format.
5533             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5534         }
5535
5536         foreach my $key (keys %$conf) {
5537             next if $key !~ m/^hostpci(\d+)$/;
5538             my $hostpciindex = $1;
5539             my $d = parse_hostpci($conf->{$key});
5540             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5541
5542             foreach my $pci (@{$d->{pciid}}) {
5543                 my $pciid = $pci->{id};
5544                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5545             }
5546         }
5547
5548         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5549     };
5550     warn $@ if $@; # avoid errors - just warn
5551 }
5552
5553 # call only in locked context
5554 sub _do_vm_stop {
5555     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5556
5557     my $pid = check_running($vmid, $nocheck);
5558     return if !$pid;
5559
5560     my $conf;
5561     if (!$nocheck) {
5562         $conf = PVE::QemuConfig->load_config($vmid);
5563         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5564         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5565             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5566             $timeout = $opts->{down} if $opts->{down};
5567         }
5568         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5569     }
5570
5571     eval {
5572         if ($shutdown) {
5573             if (defined($conf) && get_qga_key($conf, 'enabled')) {
5574                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5575             } else {
5576                 mon_cmd($vmid, "system_powerdown");
5577             }
5578         } else {
5579             mon_cmd($vmid, "quit");
5580         }
5581     };
5582     my $err = $@;
5583
5584     if (!$err) {
5585         $timeout = 60 if !defined($timeout);
5586
5587         my $count = 0;
5588         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5589             $count++;
5590             sleep 1;
5591         }
5592
5593         if ($count >= $timeout) {
5594             if ($force) {
5595                 warn "VM still running - terminating now with SIGTERM\n";
5596                 kill 15, $pid;
5597             } else {
5598                 die "VM quit/powerdown failed - got timeout\n";
5599             }
5600         } else {
5601             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5602             return;
5603         }
5604     } else {
5605        if (!check_running($vmid, $nocheck)) {
5606             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5607             return;
5608        }
5609        if ($force) {
5610             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5611             kill 15, $pid;
5612         } else {
5613             die "VM quit/powerdown failed\n";
5614         }
5615     }
5616
5617     # wait again
5618     $timeout = 10;
5619
5620     my $count = 0;
5621     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5622         $count++;
5623         sleep 1;
5624     }
5625
5626     if ($count >= $timeout) {
5627         warn "VM still running - terminating now with SIGKILL\n";
5628         kill 9, $pid;
5629         sleep 1;
5630     }
5631
5632     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5633 }
5634
5635 # Note: use $nocheck to skip tests if VM configuration file exists.
5636 # We need that when migration VMs to other nodes (files already moved)
5637 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5638 sub vm_stop {
5639     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5640
5641     $force = 1 if !defined($force) && !$shutdown;
5642
5643     if ($migratedfrom){
5644         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5645         kill 15, $pid if $pid;
5646         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5647         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5648         return;
5649     }
5650
5651     PVE::QemuConfig->lock_config($vmid, sub {
5652         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5653    });
5654 }
5655
5656 sub vm_reboot {
5657     my ($vmid, $timeout) = @_;
5658
5659     PVE::QemuConfig->lock_config($vmid, sub {
5660         eval {
5661
5662             # only reboot if running, as qmeventd starts it again on a stop event
5663             return if !check_running($vmid);
5664
5665             create_reboot_request($vmid);
5666
5667             my $storecfg = PVE::Storage::config();
5668             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5669
5670         };
5671         if (my $err = $@) {
5672             # avoid that the next normal shutdown will be confused for a reboot
5673             clear_reboot_request($vmid);
5674             die $err;
5675         }
5676    });
5677 }
5678
5679 # note: if using the statestorage parameter, the caller has to check privileges
5680 sub vm_suspend {
5681     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5682
5683     my $conf;
5684     my $path;
5685     my $storecfg;
5686     my $vmstate;
5687
5688     PVE::QemuConfig->lock_config($vmid, sub {
5689
5690         $conf = PVE::QemuConfig->load_config($vmid);
5691
5692         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5693         PVE::QemuConfig->check_lock($conf)
5694             if !($skiplock || $is_backing_up);
5695
5696         die "cannot suspend to disk during backup\n"
5697             if $is_backing_up && $includestate;
5698
5699         if ($includestate) {
5700             $conf->{lock} = 'suspending';
5701             my $date = strftime("%Y-%m-%d", localtime(time()));
5702             $storecfg = PVE::Storage::config();
5703             if (!$statestorage) {
5704                 $statestorage = find_vmstate_storage($conf, $storecfg);
5705                 # check permissions for the storage
5706                 my $rpcenv = PVE::RPCEnvironment::get();
5707                 if ($rpcenv->{type} ne 'cli') {
5708                     my $authuser = $rpcenv->get_user();
5709                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5710                 }
5711             }
5712
5713
5714             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
5715                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5716             $path = PVE::Storage::path($storecfg, $vmstate);
5717             PVE::QemuConfig->write_config($vmid, $conf);
5718         } else {
5719             mon_cmd($vmid, "stop");
5720         }
5721     });
5722
5723     if ($includestate) {
5724         # save vm state
5725         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5726
5727         eval {
5728             set_migration_caps($vmid, 1);
5729             mon_cmd($vmid, "savevm-start", statefile => $path);
5730             for(;;) {
5731                 my $state = mon_cmd($vmid, "query-savevm");
5732                 if (!$state->{status}) {
5733                     die "savevm not active\n";
5734                 } elsif ($state->{status} eq 'active') {
5735                     sleep(1);
5736                     next;
5737                 } elsif ($state->{status} eq 'completed') {
5738                     print "State saved, quitting\n";
5739                     last;
5740                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5741                     die "query-savevm failed with error '$state->{error}'\n"
5742                 } else {
5743                     die "query-savevm returned status '$state->{status}'\n";
5744                 }
5745             }
5746         };
5747         my $err = $@;
5748
5749         PVE::QemuConfig->lock_config($vmid, sub {
5750             $conf = PVE::QemuConfig->load_config($vmid);
5751             if ($err) {
5752                 # cleanup, but leave suspending lock, to indicate something went wrong
5753                 eval {
5754                     mon_cmd($vmid, "savevm-end");
5755                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5756                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5757                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
5758                     PVE::QemuConfig->write_config($vmid, $conf);
5759                 };
5760                 warn $@ if $@;
5761                 die $err;
5762             }
5763
5764             die "lock changed unexpectedly\n"
5765                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5766
5767             mon_cmd($vmid, "quit");
5768             $conf->{lock} = 'suspended';
5769             PVE::QemuConfig->write_config($vmid, $conf);
5770         });
5771     }
5772 }
5773
5774 sub vm_resume {
5775     my ($vmid, $skiplock, $nocheck) = @_;
5776
5777     PVE::QemuConfig->lock_config($vmid, sub {
5778         my $res = mon_cmd($vmid, 'query-status');
5779         my $resume_cmd = 'cont';
5780         my $reset = 0;
5781
5782         if ($res->{status}) {
5783             return if $res->{status} eq 'running'; # job done, go home
5784             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
5785             $reset = 1 if $res->{status} eq 'shutdown';
5786         }
5787
5788         if (!$nocheck) {
5789
5790             my $conf = PVE::QemuConfig->load_config($vmid);
5791
5792             PVE::QemuConfig->check_lock($conf)
5793                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5794         }
5795
5796         if ($reset) {
5797             # required if a VM shuts down during a backup and we get a resume
5798             # request before the backup finishes for example
5799             mon_cmd($vmid, "system_reset");
5800         }
5801         mon_cmd($vmid, $resume_cmd);
5802     });
5803 }
5804
5805 sub vm_sendkey {
5806     my ($vmid, $skiplock, $key) = @_;
5807
5808     PVE::QemuConfig->lock_config($vmid, sub {
5809
5810         my $conf = PVE::QemuConfig->load_config($vmid);
5811
5812         # there is no qmp command, so we use the human monitor command
5813         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5814         die $res if $res ne '';
5815     });
5816 }
5817
5818 # vzdump restore implementaion
5819
5820 sub tar_archive_read_firstfile {
5821     my $archive = shift;
5822
5823     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5824
5825     # try to detect archive type first
5826     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5827         die "unable to open file '$archive'\n";
5828     my $firstfile = <$fh>;
5829     kill 15, $pid;
5830     close $fh;
5831
5832     die "ERROR: archive contaions no data\n" if !$firstfile;
5833     chomp $firstfile;
5834
5835     return $firstfile;
5836 }
5837
5838 sub tar_restore_cleanup {
5839     my ($storecfg, $statfile) = @_;
5840
5841     print STDERR "starting cleanup\n";
5842
5843     if (my $fd = IO::File->new($statfile, "r")) {
5844         while (defined(my $line = <$fd>)) {
5845             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5846                 my $volid = $2;
5847                 eval {
5848                     if ($volid =~ m|^/|) {
5849                         unlink $volid || die 'unlink failed\n';
5850                     } else {
5851                         PVE::Storage::vdisk_free($storecfg, $volid);
5852                     }
5853                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5854                 };
5855                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5856             } else {
5857                 print STDERR "unable to parse line in statfile - $line";
5858             }
5859         }
5860         $fd->close();
5861     }
5862 }
5863
5864 sub restore_file_archive {
5865     my ($archive, $vmid, $user, $opts) = @_;
5866
5867     return restore_vma_archive($archive, $vmid, $user, $opts)
5868         if $archive eq '-';
5869
5870     my $info = PVE::Storage::archive_info($archive);
5871     my $format = $opts->{format} // $info->{format};
5872     my $comp = $info->{compression};
5873
5874     # try to detect archive format
5875     if ($format eq 'tar') {
5876         return restore_tar_archive($archive, $vmid, $user, $opts);
5877     } else {
5878         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5879     }
5880 }
5881
5882 # hepler to remove disks that will not be used after restore
5883 my $restore_cleanup_oldconf = sub {
5884     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
5885
5886     PVE::QemuConfig->foreach_volume($oldconf, sub {
5887         my ($ds, $drive) = @_;
5888
5889         return if drive_is_cdrom($drive, 1);
5890
5891         my $volid = $drive->{file};
5892         return if !$volid || $volid =~ m|^/|;
5893
5894         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
5895         return if !$path || !$owner || ($owner != $vmid);
5896
5897         # Note: only delete disk we want to restore
5898         # other volumes will become unused
5899         if ($virtdev_hash->{$ds}) {
5900             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
5901             if (my $err = $@) {
5902                 warn $err;
5903             }
5904         }
5905     });
5906
5907     # delete vmstate files, after the restore we have no snapshots anymore
5908     foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5909         my $snap = $oldconf->{snapshots}->{$snapname};
5910         if ($snap->{vmstate}) {
5911             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
5912             if (my $err = $@) {
5913                 warn $err;
5914             }
5915         }
5916     }
5917 };
5918
5919 # Helper to parse vzdump backup device hints
5920 #
5921 # $rpcenv: Environment, used to ckeck storage permissions
5922 # $user: User ID, to check storage permissions
5923 # $storecfg: Storage configuration
5924 # $fh: the file handle for reading the configuration
5925 # $devinfo: should contain device sizes for all backu-up'ed devices
5926 # $options: backup options (pool, default storage)
5927 #
5928 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
5929 my $parse_backup_hints = sub {
5930     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
5931
5932     my $virtdev_hash = {};
5933
5934     while (defined(my $line = <$fh>)) {
5935         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5936             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5937             die "archive does not contain data for drive '$virtdev'\n"
5938                 if !$devinfo->{$devname};
5939
5940             if (defined($options->{storage})) {
5941                 $storeid = $options->{storage} || 'local';
5942             } elsif (!$storeid) {
5943                 $storeid = 'local';
5944             }
5945             $format = 'raw' if !$format;
5946             $devinfo->{$devname}->{devname} = $devname;
5947             $devinfo->{$devname}->{virtdev} = $virtdev;
5948             $devinfo->{$devname}->{format} = $format;
5949             $devinfo->{$devname}->{storeid} = $storeid;
5950
5951             # check permission on storage
5952             my $pool = $options->{pool}; # todo: do we need that?
5953             if ($user ne 'root@pam') {
5954                 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5955             }
5956
5957             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5958         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
5959             my $virtdev = $1;
5960             my $drive = parse_drive($virtdev, $2);
5961             if (drive_is_cloudinit($drive)) {
5962                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5963                 $storeid = $options->{storage} if defined ($options->{storage});
5964                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5965                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
5966
5967                 $virtdev_hash->{$virtdev} = {
5968                     format => $format,
5969                     storeid => $storeid,
5970                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
5971                     is_cloudinit => 1,
5972                 };
5973             }
5974         }
5975     }
5976
5977     return $virtdev_hash;
5978 };
5979
5980 # Helper to allocate and activate all volumes required for a restore
5981 #
5982 # $storecfg: Storage configuration
5983 # $virtdev_hash: as returned by parse_backup_hints()
5984 #
5985 # Returns: { $virtdev => $volid }
5986 my $restore_allocate_devices = sub {
5987     my ($storecfg, $virtdev_hash, $vmid) = @_;
5988
5989     my $map = {};
5990     foreach my $virtdev (sort keys %$virtdev_hash) {
5991         my $d = $virtdev_hash->{$virtdev};
5992         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5993         my $storeid = $d->{storeid};
5994         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5995
5996         # test if requested format is supported
5997         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5998         my $supported = grep { $_ eq $d->{format} } @$validFormats;
5999         $d->{format} = $defFormat if !$supported;
6000
6001         my $name;
6002         if ($d->{is_cloudinit}) {
6003             $name = "vm-$vmid-cloudinit";
6004             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6005             if ($scfg->{path}) {
6006                 $name .= ".$d->{format}";
6007             }
6008         }
6009
6010         my $volid = PVE::Storage::vdisk_alloc(
6011             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6012
6013         print STDERR "new volume ID is '$volid'\n";
6014         $d->{volid} = $volid;
6015
6016         PVE::Storage::activate_volumes($storecfg, [$volid]);
6017
6018         $map->{$virtdev} = $volid;
6019     }
6020
6021     return $map;
6022 };
6023
6024 sub restore_update_config_line {
6025     my ($cookie, $map, $line, $unique) = @_;
6026
6027     return '' if $line =~ m/^\#qmdump\#/;
6028     return '' if $line =~ m/^\#vzdump\#/;
6029     return '' if $line =~ m/^lock:/;
6030     return '' if $line =~ m/^unused\d+:/;
6031     return '' if $line =~ m/^parent:/;
6032
6033     my $res = '';
6034
6035     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6036     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6037         # try to convert old 1.X settings
6038         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6039         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6040             my ($model, $macaddr) = split(/\=/, $devconfig);
6041             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6042             my $net = {
6043                 model => $model,
6044                 bridge => "vmbr$ind",
6045                 macaddr => $macaddr,
6046             };
6047             my $netstr = print_net($net);
6048
6049             $res .= "net$cookie->{netcount}: $netstr\n";
6050             $cookie->{netcount}++;
6051         }
6052     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6053         my ($id, $netstr) = ($1, $2);
6054         my $net = parse_net($netstr);
6055         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6056         $netstr = print_net($net);
6057         $res .= "$id: $netstr\n";
6058     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
6059         my $virtdev = $1;
6060         my $value = $3;
6061         my $di = parse_drive($virtdev, $value);
6062         if (defined($di->{backup}) && !$di->{backup}) {
6063             $res .= "#$line";
6064         } elsif ($map->{$virtdev}) {
6065             delete $di->{format}; # format can change on restore
6066             $di->{file} = $map->{$virtdev};
6067             $value = print_drive($di);
6068             $res .= "$virtdev: $value\n";
6069         } else {
6070             $res .= $line;
6071         }
6072     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6073         my $vmgenid = $1;
6074         if ($vmgenid ne '0') {
6075             # always generate a new vmgenid if there was a valid one setup
6076             $vmgenid = generate_uuid();
6077         }
6078         $res .= "vmgenid: $vmgenid\n";
6079     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6080         my ($uuid, $uuid_str);
6081         UUID::generate($uuid);
6082         UUID::unparse($uuid, $uuid_str);
6083         my $smbios1 = parse_smbios1($2);
6084         $smbios1->{uuid} = $uuid_str;
6085         $res .= $1.print_smbios1($smbios1)."\n";
6086     } else {
6087         $res .= $line;
6088     }
6089
6090     return $res;
6091 }
6092
6093 my $restore_deactivate_volumes = sub {
6094     my ($storecfg, $devinfo) = @_;
6095
6096     my $vollist = [];
6097     foreach my $devname (keys %$devinfo) {
6098         my $volid = $devinfo->{$devname}->{volid};
6099         push @$vollist, $volid if $volid;
6100     }
6101
6102     PVE::Storage::deactivate_volumes($storecfg, $vollist);
6103 };
6104
6105 my $restore_destroy_volumes = sub {
6106     my ($storecfg, $devinfo) = @_;
6107
6108     foreach my $devname (keys %$devinfo) {
6109         my $volid = $devinfo->{$devname}->{volid};
6110         next if !$volid;
6111         eval {
6112             if ($volid =~ m|^/|) {
6113                 unlink $volid || die 'unlink failed\n';
6114             } else {
6115                 PVE::Storage::vdisk_free($storecfg, $volid);
6116             }
6117             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6118         };
6119         print STDERR "unable to cleanup '$volid' - $@" if $@;
6120     }
6121 };
6122
6123 sub scan_volids {
6124     my ($cfg, $vmid) = @_;
6125
6126     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6127
6128     my $volid_hash = {};
6129     foreach my $storeid (keys %$info) {
6130         foreach my $item (@{$info->{$storeid}}) {
6131             next if !($item->{volid} && $item->{size});
6132             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6133             $volid_hash->{$item->{volid}} = $item;
6134         }
6135     }
6136
6137     return $volid_hash;
6138 }
6139
6140 sub update_disk_config {
6141     my ($vmid, $conf, $volid_hash) = @_;
6142
6143     my $changes;
6144     my $prefix = "VM $vmid";
6145
6146     # used and unused disks
6147     my $referenced = {};
6148
6149     # Note: it is allowed to define multiple storages with same path (alias), so
6150     # we need to check both 'volid' and real 'path' (two different volid can point
6151     # to the same path).
6152
6153     my $referencedpath = {};
6154
6155     # update size info
6156     PVE::QemuConfig->foreach_volume($conf, sub {
6157         my ($opt, $drive) = @_;
6158
6159         my $volid = $drive->{file};
6160         return if !$volid;
6161         my $volume = $volid_hash->{$volid};
6162
6163         # mark volid as "in-use" for next step
6164         $referenced->{$volid} = 1;
6165         if ($volume && (my $path = $volume->{path})) {
6166             $referencedpath->{$path} = 1;
6167         }
6168
6169         return if drive_is_cdrom($drive);
6170         return if !$volume;
6171
6172         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6173         if (defined($updated)) {
6174             $changes = 1;
6175             $conf->{$opt} = print_drive($updated);
6176             print "$prefix ($opt): $msg\n";
6177         }
6178     });
6179
6180     # remove 'unusedX' entry if volume is used
6181     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6182         my ($opt, $drive) = @_;
6183
6184         my $volid = $drive->{file};
6185         return if !$volid;
6186
6187         my $path;
6188         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6189         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6190             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6191             $changes = 1;
6192             delete $conf->{$opt};
6193         }
6194
6195         $referenced->{$volid} = 1;
6196         $referencedpath->{$path} = 1 if $path;
6197     });
6198
6199     foreach my $volid (sort keys %$volid_hash) {
6200         next if $volid =~ m/vm-$vmid-state-/;
6201         next if $referenced->{$volid};
6202         my $path = $volid_hash->{$volid}->{path};
6203         next if !$path; # just to be sure
6204         next if $referencedpath->{$path};
6205         $changes = 1;
6206         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6207         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6208         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6209     }
6210
6211     return $changes;
6212 }
6213
6214 sub rescan {
6215     my ($vmid, $nolock, $dryrun) = @_;
6216
6217     my $cfg = PVE::Storage::config();
6218
6219     print "rescan volumes...\n";
6220     my $volid_hash = scan_volids($cfg, $vmid);
6221
6222     my $updatefn =  sub {
6223         my ($vmid) = @_;
6224
6225         my $conf = PVE::QemuConfig->load_config($vmid);
6226
6227         PVE::QemuConfig->check_lock($conf);
6228
6229         my $vm_volids = {};
6230         foreach my $volid (keys %$volid_hash) {
6231             my $info = $volid_hash->{$volid};
6232             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6233         }
6234
6235         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6236
6237         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6238     };
6239
6240     if (defined($vmid)) {
6241         if ($nolock) {
6242             &$updatefn($vmid);
6243         } else {
6244             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6245         }
6246     } else {
6247         my $vmlist = config_list();
6248         foreach my $vmid (keys %$vmlist) {
6249             if ($nolock) {
6250                 &$updatefn($vmid);
6251             } else {
6252                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6253             }
6254         }
6255     }
6256 }
6257
6258 sub restore_proxmox_backup_archive {
6259     my ($archive, $vmid, $user, $options) = @_;
6260
6261     my $storecfg = PVE::Storage::config();
6262
6263     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6264     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6265
6266     my $fingerprint = $scfg->{fingerprint};
6267     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6268
6269     my $repo = PVE::PBSClient::get_repository($scfg);
6270
6271     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6272     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6273     local $ENV{PBS_PASSWORD} = $password;
6274     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6275
6276     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6277         PVE::Storage::parse_volname($storecfg, $archive);
6278
6279     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6280
6281     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6282
6283     my $tmpdir = "/var/tmp/vzdumptmp$$";
6284     rmtree $tmpdir;
6285     mkpath $tmpdir;
6286
6287     my $conffile = PVE::QemuConfig->config_file($vmid);
6288      # disable interrupts (always do cleanups)
6289     local $SIG{INT} =
6290         local $SIG{TERM} =
6291         local $SIG{QUIT} =
6292         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6293
6294     # Note: $oldconf is undef if VM does not exists
6295     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6296     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6297     my $new_conf_raw = '';
6298
6299     my $rpcenv = PVE::RPCEnvironment::get();
6300     my $devinfo = {};
6301
6302     eval {
6303         # enable interrupts
6304         local $SIG{INT} =
6305             local $SIG{TERM} =
6306             local $SIG{QUIT} =
6307             local $SIG{HUP} =
6308             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6309
6310         my $cfgfn = "$tmpdir/qemu-server.conf";
6311         my $firewall_config_fn = "$tmpdir/fw.conf";
6312         my $index_fn = "$tmpdir/index.json";
6313
6314         my $cmd = "restore";
6315
6316         my $param = [$pbs_backup_name, "index.json", $index_fn];
6317         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6318         my $index = PVE::Tools::file_get_contents($index_fn);
6319         $index = decode_json($index);
6320
6321         # print Dumper($index);
6322         foreach my $info (@{$index->{files}}) {
6323             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6324                 my $devname = $1;
6325                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6326                     $devinfo->{$devname}->{size} = $1;
6327                 } else {
6328                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6329                 }
6330             }
6331         }
6332
6333         my $is_qemu_server_backup = scalar(
6334             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6335         );
6336         if (!$is_qemu_server_backup) {
6337             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6338         }
6339         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6340
6341         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6342         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6343
6344         if ($has_firewall_config) {
6345             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6346             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6347
6348             my $pve_firewall_dir = '/etc/pve/firewall';
6349             mkdir $pve_firewall_dir; # make sure the dir exists
6350             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6351         }
6352
6353         my $fh = IO::File->new($cfgfn, "r") ||
6354             die "unable to read qemu-server.conf - $!\n";
6355
6356         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6357
6358         # fixme: rate limit?
6359
6360         # create empty/temp config
6361         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6362
6363         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6364
6365         # allocate volumes
6366         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6367
6368         foreach my $virtdev (sort keys %$virtdev_hash) {
6369             my $d = $virtdev_hash->{$virtdev};
6370             next if $d->{is_cloudinit}; # no need to restore cloudinit
6371
6372             # for live-restore we only want to preload the efidisk
6373             next if $options->{live} && $virtdev ne 'efidisk0';
6374
6375             my $volid = $d->{volid};
6376
6377             my $path = PVE::Storage::path($storecfg, $volid);
6378
6379             my $pbs_restore_cmd = [
6380                 '/usr/bin/pbs-restore',
6381                 '--repository', $repo,
6382                 $pbs_backup_name,
6383                 "$d->{devname}.img.fidx",
6384                 $path,
6385                 '--verbose',
6386                 ];
6387
6388             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6389             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
6390
6391             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6392                 push @$pbs_restore_cmd, '--skip-zero';
6393             }
6394
6395             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6396             print "restore proxmox backup image: $dbg_cmdstring\n";
6397             run_command($pbs_restore_cmd);
6398         }
6399
6400         $fh->seek(0, 0) || die "seek failed - $!\n";
6401
6402         my $cookie = { netcount => 0 };
6403         while (defined(my $line = <$fh>)) {
6404             $new_conf_raw .= restore_update_config_line(
6405                 $cookie,
6406                 $map,
6407                 $line,
6408                 $options->{unique},
6409             );
6410         }
6411
6412         $fh->close();
6413     };
6414     my $err = $@;
6415
6416     if ($err || !$options->{live}) {
6417         $restore_deactivate_volumes->($storecfg, $devinfo);
6418     }
6419
6420     rmtree $tmpdir;
6421
6422     if ($err) {
6423         $restore_destroy_volumes->($storecfg, $devinfo);
6424         die $err;
6425     }
6426
6427     if ($options->{live}) {
6428         # keep lock during live-restore
6429         $new_conf_raw .= "\nlock: create";
6430     }
6431
6432     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6433
6434     PVE::Cluster::cfs_update(); # make sure we read new file
6435
6436     eval { rescan($vmid, 1); };
6437     warn $@ if $@;
6438
6439     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
6440
6441     if ($options->{live}) {
6442         # enable interrupts
6443         local $SIG{INT} =
6444             local $SIG{TERM} =
6445             local $SIG{QUIT} =
6446             local $SIG{HUP} =
6447             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
6448
6449         my $conf = PVE::QemuConfig->load_config($vmid);
6450         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
6451
6452         delete $devinfo->{'drive-efidisk0'};
6453         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $repo, $keyfile, $pbs_backup_name);
6454
6455         PVE::QemuConfig->remove_lock($vmid, "create");
6456     }
6457 }
6458
6459 sub pbs_live_restore {
6460     my ($vmid, $conf, $storecfg, $restored_disks, $repo, $keyfile, $snap) = @_;
6461
6462     print "starting VM for live-restore\n";
6463     print "repository: '$repo', snapshot: '$snap'\n";
6464
6465     my $pbs_backing = {};
6466     for my $ds (keys %$restored_disks) {
6467         $ds =~ m/^drive-(.*)$/;
6468         my $confname = $1;
6469         $pbs_backing->{$confname} = {
6470             repository => $repo,
6471             snapshot => $snap,
6472             archive => "$ds.img.fidx",
6473         };
6474         $pbs_backing->{$confname}->{keyfile} = $keyfile if -e $keyfile;
6475
6476         my $drive = parse_drive($confname, $conf->{$confname});
6477         print "restoring '$ds' to '$drive->{file}'\n";
6478     }
6479
6480     my $drives_streamed = 0;
6481     eval {
6482         # make sure HA doesn't interrupt our restore by stopping the VM
6483         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
6484             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
6485         }
6486
6487         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
6488         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
6489         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
6490
6491         my $qmeventd_fd = register_qmeventd_handle($vmid);
6492
6493         # begin streaming, i.e. data copy from PBS to target disk for every vol,
6494         # this will effectively collapse the backing image chain consisting of
6495         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
6496         # removes itself once all backing images vanish with 'auto-remove=on')
6497         my $jobs = {};
6498         for my $ds (sort keys %$restored_disks) {
6499             my $job_id = "restore-$ds";
6500             mon_cmd($vmid, 'block-stream',
6501                 'job-id' => $job_id,
6502                 device => "$ds",
6503             );
6504             $jobs->{$job_id} = {};
6505         }
6506
6507         mon_cmd($vmid, 'cont');
6508         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
6509
6510         print "restore-drive jobs finished successfully, removing all tracking block devices"
6511             ." to disconnect from Proxmox Backup Server\n";
6512
6513         for my $ds (sort keys %$restored_disks) {
6514             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
6515         }
6516
6517         close($qmeventd_fd);
6518     };
6519
6520     my $err = $@;
6521
6522     if ($err) {
6523         warn "An error occured during live-restore: $err\n";
6524         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
6525         die "live-restore failed\n";
6526     }
6527 }
6528
6529 sub restore_vma_archive {
6530     my ($archive, $vmid, $user, $opts, $comp) = @_;
6531
6532     my $readfrom = $archive;
6533
6534     my $cfg = PVE::Storage::config();
6535     my $commands = [];
6536     my $bwlimit = $opts->{bwlimit};
6537
6538     my $dbg_cmdstring = '';
6539     my $add_pipe = sub {
6540         my ($cmd) = @_;
6541         push @$commands, $cmd;
6542         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6543         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6544         $readfrom = '-';
6545     };
6546
6547     my $input = undef;
6548     if ($archive eq '-') {
6549         $input = '<&STDIN';
6550     } else {
6551         # If we use a backup from a PVE defined storage we also consider that
6552         # storage's rate limit:
6553         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6554         if (defined($volid)) {
6555             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6556             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6557             if ($readlimit) {
6558                 print STDERR "applying read rate limit: $readlimit\n";
6559                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6560                 $add_pipe->($cstream);
6561             }
6562         }
6563     }
6564
6565     if ($comp) {
6566         my $info = PVE::Storage::decompressor_info('vma', $comp);
6567         my $cmd = $info->{decompressor};
6568         push @$cmd, $readfrom;
6569         $add_pipe->($cmd);
6570     }
6571
6572     my $tmpdir = "/var/tmp/vzdumptmp$$";
6573     rmtree $tmpdir;
6574
6575     # disable interrupts (always do cleanups)
6576     local $SIG{INT} =
6577         local $SIG{TERM} =
6578         local $SIG{QUIT} =
6579         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6580
6581     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6582     POSIX::mkfifo($mapfifo, 0600);
6583     my $fifofh;
6584     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
6585
6586     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6587
6588     my $oldtimeout;
6589     my $timeout = 5;
6590
6591     my $devinfo = {};
6592
6593     my $rpcenv = PVE::RPCEnvironment::get();
6594
6595     my $conffile = PVE::QemuConfig->config_file($vmid);
6596
6597     # Note: $oldconf is undef if VM does not exist
6598     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6599     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6600     my $new_conf_raw = '';
6601
6602     my %storage_limits;
6603
6604     my $print_devmap = sub {
6605         my $cfgfn = "$tmpdir/qemu-server.conf";
6606
6607         # we can read the config - that is already extracted
6608         my $fh = IO::File->new($cfgfn, "r") ||
6609             die "unable to read qemu-server.conf - $!\n";
6610
6611         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6612         if (-f $fwcfgfn) {
6613             my $pve_firewall_dir = '/etc/pve/firewall';
6614             mkdir $pve_firewall_dir; # make sure the dir exists
6615             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6616         }
6617
6618         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6619
6620         foreach my $info (values %{$virtdev_hash}) {
6621             my $storeid = $info->{storeid};
6622             next if defined($storage_limits{$storeid});
6623
6624             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
6625             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
6626             $storage_limits{$storeid} = $limit * 1024;
6627         }
6628
6629         foreach my $devname (keys %$devinfo) {
6630             die "found no device mapping information for device '$devname'\n"
6631                 if !$devinfo->{$devname}->{virtdev};
6632         }
6633
6634         # create empty/temp config
6635         if ($oldconf) {
6636             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6637             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6638         }
6639
6640         # allocate volumes
6641         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6642
6643         # print restore information to $fifofh
6644         foreach my $virtdev (sort keys %$virtdev_hash) {
6645             my $d = $virtdev_hash->{$virtdev};
6646             next if $d->{is_cloudinit}; # no need to restore cloudinit
6647
6648             my $storeid = $d->{storeid};
6649             my $volid = $d->{volid};
6650
6651             my $map_opts = '';
6652             if (my $limit = $storage_limits{$storeid}) {
6653                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6654             }
6655
6656             my $write_zeros = 1;
6657             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6658                 $write_zeros = 0;
6659             }
6660
6661             my $path = PVE::Storage::path($cfg, $volid);
6662
6663             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6664
6665             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6666         }
6667
6668         $fh->seek(0, 0) || die "seek failed - $!\n";
6669
6670         my $cookie = { netcount => 0 };
6671         while (defined(my $line = <$fh>)) {
6672             $new_conf_raw .= restore_update_config_line(
6673                 $cookie,
6674                 $map,
6675                 $line,
6676                 $opts->{unique},
6677             );
6678         }
6679
6680         $fh->close();
6681     };
6682
6683     eval {
6684         # enable interrupts
6685         local $SIG{INT} =
6686             local $SIG{TERM} =
6687             local $SIG{QUIT} =
6688             local $SIG{HUP} =
6689             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6690         local $SIG{ALRM} = sub { die "got timeout\n"; };
6691
6692         $oldtimeout = alarm($timeout);
6693
6694         my $parser = sub {
6695             my $line = shift;
6696
6697             print "$line\n";
6698
6699             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6700                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6701                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6702             } elsif ($line =~ m/^CTIME: /) {
6703                 # we correctly received the vma config, so we can disable
6704                 # the timeout now for disk allocation (set to 10 minutes, so
6705                 # that we always timeout if something goes wrong)
6706                 alarm(600);
6707                 &$print_devmap();
6708                 print $fifofh "done\n";
6709                 my $tmp = $oldtimeout || 0;
6710                 $oldtimeout = undef;
6711                 alarm($tmp);
6712                 close($fifofh);
6713                 $fifofh = undef;
6714             }
6715         };
6716
6717         print "restore vma archive: $dbg_cmdstring\n";
6718         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6719     };
6720     my $err = $@;
6721
6722     alarm($oldtimeout) if $oldtimeout;
6723
6724     $restore_deactivate_volumes->($cfg, $devinfo);
6725
6726     close($fifofh) if $fifofh;
6727     unlink $mapfifo;
6728     rmtree $tmpdir;
6729
6730     if ($err) {
6731         $restore_destroy_volumes->($cfg, $devinfo);
6732         die $err;
6733     }
6734
6735     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6736
6737     PVE::Cluster::cfs_update(); # make sure we read new file
6738
6739     eval { rescan($vmid, 1); };
6740     warn $@ if $@;
6741
6742     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
6743 }
6744
6745 sub restore_tar_archive {
6746     my ($archive, $vmid, $user, $opts) = @_;
6747
6748     if ($archive ne '-') {
6749         my $firstfile = tar_archive_read_firstfile($archive);
6750         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
6751             if $firstfile ne 'qemu-server.conf';
6752     }
6753
6754     my $storecfg = PVE::Storage::config();
6755
6756     # avoid zombie disks when restoring over an existing VM -> cleanup first
6757     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6758     # skiplock=1 because qmrestore has set the 'create' lock itself already
6759     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6760     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6761
6762     my $tocmd = "/usr/lib/qemu-server/qmextract";
6763
6764     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6765     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6766     $tocmd .= ' --prealloc' if $opts->{prealloc};
6767     $tocmd .= ' --info' if $opts->{info};
6768
6769     # tar option "xf" does not autodetect compression when read from STDIN,
6770     # so we pipe to zcat
6771     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6772         PVE::Tools::shellquote("--to-command=$tocmd");
6773
6774     my $tmpdir = "/var/tmp/vzdumptmp$$";
6775     mkpath $tmpdir;
6776
6777     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6778     local $ENV{VZDUMP_VMID} = $vmid;
6779     local $ENV{VZDUMP_USER} = $user;
6780
6781     my $conffile = PVE::QemuConfig->config_file($vmid);
6782     my $new_conf_raw = '';
6783
6784     # disable interrupts (always do cleanups)
6785     local $SIG{INT} =
6786         local $SIG{TERM} =
6787         local $SIG{QUIT} =
6788         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6789
6790     eval {
6791         # enable interrupts
6792         local $SIG{INT} =
6793             local $SIG{TERM} =
6794             local $SIG{QUIT} =
6795             local $SIG{HUP} =
6796             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6797
6798         if ($archive eq '-') {
6799             print "extracting archive from STDIN\n";
6800             run_command($cmd, input => "<&STDIN");
6801         } else {
6802             print "extracting archive '$archive'\n";
6803             run_command($cmd);
6804         }
6805
6806         return if $opts->{info};
6807
6808         # read new mapping
6809         my $map = {};
6810         my $statfile = "$tmpdir/qmrestore.stat";
6811         if (my $fd = IO::File->new($statfile, "r")) {
6812             while (defined (my $line = <$fd>)) {
6813                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6814                     $map->{$1} = $2 if $1;
6815                 } else {
6816                     print STDERR "unable to parse line in statfile - $line\n";
6817                 }
6818             }
6819             $fd->close();
6820         }
6821
6822         my $confsrc = "$tmpdir/qemu-server.conf";
6823
6824         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
6825
6826         my $cookie = { netcount => 0 };
6827         while (defined (my $line = <$srcfd>)) {
6828             $new_conf_raw .= restore_update_config_line(
6829                 $cookie,
6830                 $map,
6831                 $line,
6832                 $opts->{unique},
6833             );
6834         }
6835
6836         $srcfd->close();
6837     };
6838     if (my $err = $@) {
6839         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6840         die $err;
6841     }
6842
6843     rmtree $tmpdir;
6844
6845     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6846
6847     PVE::Cluster::cfs_update(); # make sure we read new file
6848
6849     eval { rescan($vmid, 1); };
6850     warn $@ if $@;
6851 };
6852
6853 sub foreach_storage_used_by_vm {
6854     my ($conf, $func) = @_;
6855
6856     my $sidhash = {};
6857
6858     PVE::QemuConfig->foreach_volume($conf, sub {
6859         my ($ds, $drive) = @_;
6860         return if drive_is_cdrom($drive);
6861
6862         my $volid = $drive->{file};
6863
6864         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6865         $sidhash->{$sid} = $sid if $sid;
6866     });
6867
6868     foreach my $sid (sort keys %$sidhash) {
6869         &$func($sid);
6870     }
6871 }
6872
6873 my $qemu_snap_storage = {
6874     rbd => 1,
6875 };
6876 sub do_snapshots_with_qemu {
6877     my ($storecfg, $volid) = @_;
6878
6879     my $storage_name = PVE::Storage::parse_volume_id($volid);
6880     my $scfg = $storecfg->{ids}->{$storage_name};
6881     die "could not find storage '$storage_name'\n" if !defined($scfg);
6882
6883     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6884         return 1;
6885     }
6886
6887     if ($volid =~ m/\.(qcow2|qed)$/){
6888         return 1;
6889     }
6890
6891     return;
6892 }
6893
6894 sub qga_check_running {
6895     my ($vmid, $nowarn) = @_;
6896
6897     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6898     if ($@) {
6899         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6900         return 0;
6901     }
6902     return 1;
6903 }
6904
6905 sub template_create {
6906     my ($vmid, $conf, $disk) = @_;
6907
6908     my $storecfg = PVE::Storage::config();
6909
6910     PVE::QemuConfig->foreach_volume($conf, sub {
6911         my ($ds, $drive) = @_;
6912
6913         return if drive_is_cdrom($drive);
6914         return if $disk && $ds ne $disk;
6915
6916         my $volid = $drive->{file};
6917         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6918
6919         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6920         $drive->{file} = $voliddst;
6921         $conf->{$ds} = print_drive($drive);
6922         PVE::QemuConfig->write_config($vmid, $conf);
6923     });
6924 }
6925
6926 sub convert_iscsi_path {
6927     my ($path) = @_;
6928
6929     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6930         my $portal = $1;
6931         my $target = $2;
6932         my $lun = $3;
6933
6934         my $initiator_name = get_initiator_name();
6935
6936         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6937                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6938     }
6939
6940     die "cannot convert iscsi path '$path', unkown format\n";
6941 }
6942
6943 sub qemu_img_convert {
6944     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6945
6946     my $storecfg = PVE::Storage::config();
6947     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6948     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6949
6950     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6951
6952     my $cachemode;
6953     my $src_path;
6954     my $src_is_iscsi = 0;
6955     my $src_format;
6956
6957     if ($src_storeid) {
6958         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6959         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6960         $src_format = qemu_img_format($src_scfg, $src_volname);
6961         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6962         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6963         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6964     } elsif (-f $src_volid) {
6965         $src_path = $src_volid;
6966         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6967             $src_format = $1;
6968         }
6969     }
6970
6971     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6972
6973     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6974     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6975     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6976     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6977
6978     my $cmd = [];
6979     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6980     push @$cmd, '-l', "snapshot.name=$snapname"
6981         if $snapname && $src_format && $src_format eq "qcow2";
6982     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6983     push @$cmd, '-T', $cachemode if defined($cachemode);
6984
6985     if ($src_is_iscsi) {
6986         push @$cmd, '--image-opts';
6987         $src_path = convert_iscsi_path($src_path);
6988     } elsif ($src_format) {
6989         push @$cmd, '-f', $src_format;
6990     }
6991
6992     if ($dst_is_iscsi) {
6993         push @$cmd, '--target-image-opts';
6994         $dst_path = convert_iscsi_path($dst_path);
6995     } else {
6996         push @$cmd, '-O', $dst_format;
6997     }
6998
6999     push @$cmd, $src_path;
7000
7001     if (!$dst_is_iscsi && $is_zero_initialized) {
7002         push @$cmd, "zeroinit:$dst_path";
7003     } else {
7004         push @$cmd, $dst_path;
7005     }
7006
7007     my $parser = sub {
7008         my $line = shift;
7009         if($line =~ m/\((\S+)\/100\%\)/){
7010             my $percent = $1;
7011             my $transferred = int($size * $percent / 100);
7012             my $total_h = render_bytes($size, 1);
7013             my $transferred_h = render_bytes($transferred, 1);
7014
7015             print "transferred $transferred_h of $total_h ($percent%)\n";
7016         }
7017
7018     };
7019
7020     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7021     my $err = $@;
7022     die "copy failed: $err" if $err;
7023 }
7024
7025 sub qemu_img_format {
7026     my ($scfg, $volname) = @_;
7027
7028     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7029         return $1;
7030     } else {
7031         return "raw";
7032     }
7033 }
7034
7035 sub qemu_drive_mirror {
7036     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7037
7038     $jobs = {} if !$jobs;
7039
7040     my $qemu_target;
7041     my $format;
7042     $jobs->{"drive-$drive"} = {};
7043
7044     if ($dst_volid =~ /^nbd:/) {
7045         $qemu_target = $dst_volid;
7046         $format = "nbd";
7047     } else {
7048         my $storecfg = PVE::Storage::config();
7049         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7050
7051         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7052
7053         $format = qemu_img_format($dst_scfg, $dst_volname);
7054
7055         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7056
7057         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7058     }
7059
7060     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7061     $opts->{format} = $format if $format;
7062
7063     if (defined($src_bitmap)) {
7064         $opts->{sync} = 'incremental';
7065         $opts->{bitmap} = $src_bitmap;
7066         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7067     }
7068
7069     if (defined($bwlimit)) {
7070         $opts->{speed} = $bwlimit * 1024;
7071         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7072     } else {
7073         print "drive mirror is starting for drive-$drive\n";
7074     }
7075
7076     # if a job already runs for this device we get an error, catch it for cleanup
7077     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7078     if (my $err = $@) {
7079         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7080         warn "$@\n" if $@;
7081         die "mirroring error: $err\n";
7082     }
7083
7084     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7085 }
7086
7087 # $completion can be either
7088 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7089 # 'cancel': wait until all jobs are ready, block-job-cancel them
7090 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7091 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7092 sub qemu_drive_mirror_monitor {
7093     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7094
7095     $completion //= 'complete';
7096     $op //= "mirror";
7097
7098     eval {
7099         my $err_complete = 0;
7100
7101         my $starttime = time ();
7102         while (1) {
7103             die "block job ('$op') timed out\n" if $err_complete > 300;
7104
7105             my $stats = mon_cmd($vmid, "query-block-jobs");
7106             my $ctime = time();
7107
7108             my $running_jobs = {};
7109             for my $stat (@$stats) {
7110                 next if $stat->{type} ne $op;
7111                 $running_jobs->{$stat->{device}} = $stat;
7112             }
7113
7114             my $readycounter = 0;
7115
7116             for my $job_id (sort keys %$jobs) {
7117                 my $job = $running_jobs->{$job_id};
7118
7119                 my $vanished = !defined($job);
7120                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7121                 if($complete || ($vanished && $completion eq 'auto')) {
7122                     print "$job_id: $op-job finished\n";
7123                     delete $jobs->{$job_id};
7124                     next;
7125                 }
7126
7127                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7128
7129                 my $busy = $job->{busy};
7130                 my $ready = $job->{ready};
7131                 if (my $total = $job->{len}) {
7132                     my $transferred = $job->{offset} || 0;
7133                     my $remaining = $total - $transferred;
7134                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7135
7136                     my $duration = $ctime - $starttime;
7137                     my $total_h = render_bytes($total, 1);
7138                     my $transferred_h = render_bytes($transferred, 1);
7139
7140                     my $status = sprintf(
7141                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7142                         render_duration($duration),
7143                     );
7144
7145                     if ($ready) {
7146                         if ($busy) {
7147                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7148                         } else {
7149                             $status .= ", ready";
7150                         }
7151                     }
7152                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7153                     $jobs->{$job_id}->{ready} = $ready;
7154                 }
7155
7156                 $readycounter++ if $job->{ready};
7157             }
7158
7159             last if scalar(keys %$jobs) == 0;
7160
7161             if ($readycounter == scalar(keys %$jobs)) {
7162                 print "all '$op' jobs are ready\n";
7163
7164                 # do the complete later (or has already been done)
7165                 last if $completion eq 'skip' || $completion eq 'auto';
7166
7167                 if ($vmiddst && $vmiddst != $vmid) {
7168                     my $agent_running = $qga && qga_check_running($vmid);
7169                     if ($agent_running) {
7170                         print "freeze filesystem\n";
7171                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7172                     } else {
7173                         print "suspend vm\n";
7174                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7175                     }
7176
7177                     # if we clone a disk for a new target vm, we don't switch the disk
7178                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7179
7180                     if ($agent_running) {
7181                         print "unfreeze filesystem\n";
7182                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7183                     } else {
7184                         print "resume vm\n";
7185                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
7186                     }
7187
7188                     last;
7189                 } else {
7190
7191                     for my $job_id (sort keys %$jobs) {
7192                         # try to switch the disk if source and destination are on the same guest
7193                         print "$job_id: Completing block job_id...\n";
7194
7195                         my $op;
7196                         if ($completion eq 'complete') {
7197                             $op = 'block-job-complete';
7198                         } elsif ($completion eq 'cancel') {
7199                             $op = 'block-job-cancel';
7200                         } else {
7201                             die "invalid completion value: $completion\n";
7202                         }
7203                         eval { mon_cmd($vmid, $op, device => $job_id) };
7204                         if ($@ =~ m/cannot be completed/) {
7205                             print "$job_id: block job cannot be completed, trying again.\n";
7206                             $err_complete++;
7207                         }else {
7208                             print "$job_id: Completed successfully.\n";
7209                             $jobs->{$job_id}->{complete} = 1;
7210                         }
7211                     }
7212                 }
7213             }
7214             sleep 1;
7215         }
7216     };
7217     my $err = $@;
7218
7219     if ($err) {
7220         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7221         die "block job ($op) error: $err";
7222     }
7223 }
7224
7225 sub qemu_blockjobs_cancel {
7226     my ($vmid, $jobs) = @_;
7227
7228     foreach my $job (keys %$jobs) {
7229         print "$job: Cancelling block job\n";
7230         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7231         $jobs->{$job}->{cancel} = 1;
7232     }
7233
7234     while (1) {
7235         my $stats = mon_cmd($vmid, "query-block-jobs");
7236
7237         my $running_jobs = {};
7238         foreach my $stat (@$stats) {
7239             $running_jobs->{$stat->{device}} = $stat;
7240         }
7241
7242         foreach my $job (keys %$jobs) {
7243
7244             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7245                 print "$job: Done.\n";
7246                 delete $jobs->{$job};
7247             }
7248         }
7249
7250         last if scalar(keys %$jobs) == 0;
7251
7252         sleep 1;
7253     }
7254 }
7255
7256 sub clone_disk {
7257     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7258         $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
7259
7260     my $newvolid;
7261
7262     if (!$full) {
7263         print "create linked clone of drive $drivename ($drive->{file})\n";
7264         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7265         push @$newvollist, $newvolid;
7266     } else {
7267
7268         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7269         $storeid = $storage if $storage;
7270
7271         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7272
7273         print "create full clone of drive $drivename ($drive->{file})\n";
7274         my $name = undef;
7275         my $size = undef;
7276         if (drive_is_cloudinit($drive)) {
7277             $name = "vm-$newvmid-cloudinit";
7278             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7279             if ($scfg->{path}) {
7280                 $name .= ".$dst_format";
7281             }
7282             $snapname = undef;
7283             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7284         } elsif ($drivename eq 'efidisk0') {
7285             $size = get_efivars_size($conf);
7286         } else {
7287             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
7288         }
7289         $newvolid = PVE::Storage::vdisk_alloc(
7290             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
7291         );
7292         push @$newvollist, $newvolid;
7293
7294         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7295
7296         if (drive_is_cloudinit($drive)) {
7297             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
7298             # if this is the case, we have to complete any block-jobs still there from
7299             # previous drive-mirrors
7300             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
7301                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
7302             }
7303             goto no_data_clone;
7304         }
7305
7306         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7307         if (!$running || $snapname) {
7308             # TODO: handle bwlimits
7309             if ($drivename eq 'efidisk0') {
7310                 # the relevant data on the efidisk may be smaller than the source
7311                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
7312                 # that is given by the OVMF_VARS.fd
7313                 my $src_path = PVE::Storage::path($storecfg, $drive->{file});
7314                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
7315
7316                 # better for Ceph if block size is not too small, see bug #3324
7317                 my $bs = 1024*1024;
7318
7319                 run_command(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=$bs", "osize=$size",
7320                     "if=$src_path", "of=$dst_path"]);
7321             } else {
7322                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7323             }
7324         } else {
7325
7326             my $kvmver = get_running_qemu_version ($vmid);
7327             if (!min_version($kvmver, 2, 7)) {
7328                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7329                     if $drive->{iothread};
7330             }
7331
7332             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7333                 $completion, $qga, $bwlimit);
7334         }
7335     }
7336
7337 no_data_clone:
7338     my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
7339
7340     my $disk = $drive;
7341     $disk->{format} = undef;
7342     $disk->{file} = $newvolid;
7343     $disk->{size} = $size if defined($size);
7344
7345     return $disk;
7346 }
7347
7348 sub get_running_qemu_version {
7349     my ($vmid) = @_;
7350     my $res = mon_cmd($vmid, "query-version");
7351     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7352 }
7353
7354 sub qemu_use_old_bios_files {
7355     my ($machine_type) = @_;
7356
7357     return if !$machine_type;
7358
7359     my $use_old_bios_files = undef;
7360
7361     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7362         $machine_type = $1;
7363         $use_old_bios_files = 1;
7364     } else {
7365         my $version = extract_version($machine_type, kvm_user_version());
7366         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7367         # load new efi bios files on migration. So this hack is required to allow
7368         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7369         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7370         $use_old_bios_files = !min_version($version, 2, 4);
7371     }
7372
7373     return ($use_old_bios_files, $machine_type);
7374 }
7375
7376 sub get_efivars_size {
7377     my ($conf) = @_;
7378     my $arch = get_vm_arch($conf);
7379     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7380     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
7381     return -s $ovmf_vars;
7382 }
7383
7384 sub update_efidisk_size {
7385     my ($conf) = @_;
7386
7387     return if !defined($conf->{efidisk0});
7388
7389     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
7390     $disk->{size} = get_efivars_size($conf);
7391     $conf->{efidisk0} = print_drive($disk);
7392
7393     return;
7394 }
7395
7396 sub create_efidisk($$$$$) {
7397     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7398
7399     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7400     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7401
7402     my $vars_size_b = -s $ovmf_vars;
7403     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7404     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7405     PVE::Storage::activate_volumes($storecfg, [$volid]);
7406
7407     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7408     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7409
7410     return ($volid, $size/1024);
7411 }
7412
7413 sub vm_iothreads_list {
7414     my ($vmid) = @_;
7415
7416     my $res = mon_cmd($vmid, 'query-iothreads');
7417
7418     my $iothreads = {};
7419     foreach my $iothread (@$res) {
7420         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7421     }
7422
7423     return $iothreads;
7424 }
7425
7426 sub scsihw_infos {
7427     my ($conf, $drive) = @_;
7428
7429     my $maxdev = 0;
7430
7431     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7432         $maxdev = 7;
7433     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7434         $maxdev = 1;
7435     } else {
7436         $maxdev = 256;
7437     }
7438
7439     my $controller = int($drive->{index} / $maxdev);
7440     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
7441         ? "virtioscsi"
7442         : "scsihw";
7443
7444     return ($maxdev, $controller, $controller_prefix);
7445 }
7446
7447 sub windows_version {
7448     my ($ostype) = @_;
7449
7450     return 0 if !$ostype;
7451
7452     my $winversion = 0;
7453
7454     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7455         $winversion = 5;
7456     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7457         $winversion = 6;
7458     } elsif ($ostype =~ m/^win(\d+)$/) {
7459         $winversion = $1;
7460     }
7461
7462     return $winversion;
7463 }
7464
7465 sub resolve_dst_disk_format {
7466         my ($storecfg, $storeid, $src_volname, $format) = @_;
7467         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7468
7469         if (!$format) {
7470             # if no target format is specified, use the source disk format as hint
7471             if ($src_volname) {
7472                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7473                 $format = qemu_img_format($scfg, $src_volname);
7474             } else {
7475                 return $defFormat;
7476             }
7477         }
7478
7479         # test if requested format is supported - else use default
7480         my $supported = grep { $_ eq $format } @$validFormats;
7481         $format = $defFormat if !$supported;
7482         return $format;
7483 }
7484
7485 # NOTE: if this logic changes, please update docs & possibly gui logic
7486 sub find_vmstate_storage {
7487     my ($conf, $storecfg) = @_;
7488
7489     # first, return storage from conf if set
7490     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7491
7492     my ($target, $shared, $local);
7493
7494     foreach_storage_used_by_vm($conf, sub {
7495         my ($sid) = @_;
7496         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7497         my $dst = $scfg->{shared} ? \$shared : \$local;
7498         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7499     });
7500
7501     # second, use shared storage where VM has at least one disk
7502     # third, use local storage where VM has at least one disk
7503     # fall back to local storage
7504     $target = $shared // $local // 'local';
7505
7506     return $target;
7507 }
7508
7509 sub generate_uuid {
7510     my ($uuid, $uuid_str);
7511     UUID::generate($uuid);
7512     UUID::unparse($uuid, $uuid_str);
7513     return $uuid_str;
7514 }
7515
7516 sub generate_smbios1_uuid {
7517     return "uuid=".generate_uuid();
7518 }
7519
7520 sub nbd_stop {
7521     my ($vmid) = @_;
7522
7523     mon_cmd($vmid, 'nbd-server-stop');
7524 }
7525
7526 sub create_reboot_request {
7527     my ($vmid) = @_;
7528     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7529         or die "failed to create reboot trigger file: $!\n";
7530     close($fh);
7531 }
7532
7533 sub clear_reboot_request {
7534     my ($vmid) = @_;
7535     my $path = "/run/qemu-server/$vmid.reboot";
7536     my $res = 0;
7537
7538     $res = unlink($path);
7539     die "could not remove reboot request for $vmid: $!"
7540         if !$res && $! != POSIX::ENOENT;
7541
7542     return $res;
7543 }
7544
7545 sub bootorder_from_legacy {
7546     my ($conf, $bootcfg) = @_;
7547
7548     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
7549     my $bootindex_hash = {};
7550     my $i = 1;
7551     foreach my $o (split(//, $boot)) {
7552         $bootindex_hash->{$o} = $i*100;
7553         $i++;
7554     }
7555
7556     my $bootorder = {};
7557
7558     PVE::QemuConfig->foreach_volume($conf, sub {
7559         my ($ds, $drive) = @_;
7560
7561         if (drive_is_cdrom ($drive, 1)) {
7562             if ($bootindex_hash->{d}) {
7563                 $bootorder->{$ds} = $bootindex_hash->{d};
7564                 $bootindex_hash->{d} += 1;
7565             }
7566         } elsif ($bootindex_hash->{c}) {
7567             $bootorder->{$ds} = $bootindex_hash->{c}
7568                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
7569             $bootindex_hash->{c} += 1;
7570         }
7571     });
7572
7573     if ($bootindex_hash->{n}) {
7574         for (my $i = 0; $i < $MAX_NETS; $i++) {
7575             my $netname = "net$i";
7576             next if !$conf->{$netname};
7577             $bootorder->{$netname} = $bootindex_hash->{n};
7578             $bootindex_hash->{n} += 1;
7579         }
7580     }
7581
7582     return $bootorder;
7583 }
7584
7585 # Generate default device list for 'boot: order=' property. Matches legacy
7586 # default boot order, but with explicit device names. This is important, since
7587 # the fallback for when neither 'order' nor the old format is specified relies
7588 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
7589 sub get_default_bootdevices {
7590     my ($conf) = @_;
7591
7592     my @ret = ();
7593
7594     # harddisk
7595     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
7596     push @ret, $first if $first;
7597
7598     # cdrom
7599     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
7600     push @ret, $first if $first;
7601
7602     # network
7603     for (my $i = 0; $i < $MAX_NETS; $i++) {
7604         my $netname = "net$i";
7605         next if !$conf->{$netname};
7606         push @ret, $netname;
7607         last;
7608     }
7609
7610     return \@ret;
7611 }
7612
7613 sub device_bootorder {
7614     my ($conf) = @_;
7615
7616     return bootorder_from_legacy($conf) if !defined($conf->{boot});
7617
7618     my $boot = parse_property_string($boot_fmt, $conf->{boot});
7619
7620     my $bootorder = {};
7621     if (!defined($boot) || $boot->{legacy}) {
7622         $bootorder = bootorder_from_legacy($conf, $boot);
7623     } elsif ($boot->{order}) {
7624         my $i = 100; # start at 100 to allow user to insert devices before us with -args
7625         for my $dev (PVE::Tools::split_list($boot->{order})) {
7626             $bootorder->{$dev} = $i++;
7627         }
7628     }
7629
7630     return $bootorder;
7631 }
7632
7633 sub register_qmeventd_handle {
7634     my ($vmid) = @_;
7635
7636     my $fh;
7637     my $peer = "/var/run/qmeventd.sock";
7638     my $count = 0;
7639
7640     for (;;) {
7641         $count++;
7642         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
7643         last if $fh;
7644         if ($! != EINTR && $! != EAGAIN) {
7645             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
7646         }
7647         if ($count > 4) {
7648             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
7649               . "after $count retries\n";
7650         }
7651         usleep(25000);
7652     }
7653
7654     # send handshake to mark VM as backing up
7655     print $fh to_json({vzdump => {vmid => "$vmid"}});
7656
7657     # return handle to be closed later when inhibit is no longer required
7658     return $fh;
7659 }
7660
7661 # bash completion helper
7662
7663 sub complete_backup_archives {
7664     my ($cmdname, $pname, $cvalue) = @_;
7665
7666     my $cfg = PVE::Storage::config();
7667
7668     my $storeid;
7669
7670     if ($cvalue =~ m/^([^:]+):/) {
7671         $storeid = $1;
7672     }
7673
7674     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7675
7676     my $res = [];
7677     foreach my $id (keys %$data) {
7678         foreach my $item (@{$data->{$id}}) {
7679             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
7680             push @$res, $item->{volid} if defined($item->{volid});
7681         }
7682     }
7683
7684     return $res;
7685 }
7686
7687 my $complete_vmid_full = sub {
7688     my ($running) = @_;
7689
7690     my $idlist = vmstatus();
7691
7692     my $res = [];
7693
7694     foreach my $id (keys %$idlist) {
7695         my $d = $idlist->{$id};
7696         if (defined($running)) {
7697             next if $d->{template};
7698             next if $running && $d->{status} ne 'running';
7699             next if !$running && $d->{status} eq 'running';
7700         }
7701         push @$res, $id;
7702
7703     }
7704     return $res;
7705 };
7706
7707 sub complete_vmid {
7708     return &$complete_vmid_full();
7709 }
7710
7711 sub complete_vmid_stopped {
7712     return &$complete_vmid_full(0);
7713 }
7714
7715 sub complete_vmid_running {
7716     return &$complete_vmid_full(1);
7717 }
7718
7719 sub complete_storage {
7720
7721     my $cfg = PVE::Storage::config();
7722     my $ids = $cfg->{ids};
7723
7724     my $res = [];
7725     foreach my $sid (keys %$ids) {
7726         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7727         next if !$ids->{$sid}->{content}->{images};
7728         push @$res, $sid;
7729     }
7730
7731     return $res;
7732 }
7733
7734 sub complete_migration_storage {
7735     my ($cmd, $param, $current_value, $all_args) = @_;
7736
7737     my $targetnode = @$all_args[1];
7738
7739     my $cfg = PVE::Storage::config();
7740     my $ids = $cfg->{ids};
7741
7742     my $res = [];
7743     foreach my $sid (keys %$ids) {
7744         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7745         next if !$ids->{$sid}->{content}->{images};
7746         push @$res, $sid;
7747     }
7748
7749     return $res;
7750 }
7751
7752 sub vm_is_paused {
7753     my ($vmid) = @_;
7754     my $qmpstatus = eval {
7755         PVE::QemuConfig::assert_config_exists_on_node($vmid);
7756         mon_cmd($vmid, "query-status");
7757     };
7758     warn "$@\n" if $@;
7759     return $qmpstatus && $qmpstatus->{status} eq "paused";
7760 }
7761
7762 sub check_volume_storage_type {
7763     my ($storecfg, $vol) = @_;
7764
7765     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
7766     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7767     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
7768
7769     die "storage '$storeid' does not support content-type '$vtype'\n"
7770         if !$scfg->{content}->{$vtype};
7771
7772     return 1;
7773 }
7774
7775 1;