PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::CGroup;
  31 use PVE::DataCenterConfig;
  32 use PVE::Exception qw(raise raise_param_exc);
  33 use PVE::Format qw(render_duration render_bytes);
  34 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  35 use PVE::INotify;
  36 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  37 use PVE::ProcFSTools;
  38 use PVE::PBSClient;
  39 use PVE::RPCEnvironment;
  40 use PVE::Storage;
  41 use PVE::SysFSTools;
  42 use PVE::Systemd;
  43 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  44
  45 use PVE::QMPClient;
  46 use PVE::QemuConfig;
  47 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  48 use PVE::QemuServer::Cloudinit;
  49 use PVE::QemuServer::CGroup;
  50 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  51 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom parse_drive print_drive);
  52 use PVE::QemuServer::Machine;
  53 use PVE::QemuServer::Memory;
  54 use PVE::QemuServer::Monitor qw(mon_cmd);
  55 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  56 use PVE::QemuServer::USB qw(parse_usb_device);
  57
  58 my $have_sdn;
  59 eval {
  60     require PVE::Network::SDN::Zones;
  61     $have_sdn = 1;
  62 };
  63
  64 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  65 my $OVMF = {
  66     x86_64 => [
  67         "$EDK2_FW_BASE/OVMF_CODE.fd",
  68         "$EDK2_FW_BASE/OVMF_VARS.fd"
  69     ],
  70     aarch64 => [
  71         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  72         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  73     ],
  74 };
  75
  76 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  77
  78 # Note about locking: we use flock on the config file protect
  79 # against concurent actions.
  80 # Aditionaly, we have a 'lock' setting in the config file. This
  81 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  82 # allowed when such lock is set. But you can ignore this kind of
  83 # lock with the --skiplock flag.
  84
  85 cfs_register_file('/qemu-server/',
  86                   \&parse_vm_config,
  87                   \&write_vm_config);
  88
  89 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  90     description => "Some command save/restore state from this location.",
  91     type => 'string',
  92     maxLength => 128,
  93     optional => 1,
  94 });
  95
  96 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  97         description => "Specifies the Qemu machine type.",
  98         type => 'string',
  99         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 100         maxLength => 40,
 101         optional => 1,
 102 });
 103
 104
 105 sub map_storage {
 106     my ($map, $source) = @_;
 107
 108     return $source if !defined($map);
 109
 110     return $map->{entries}->{$source}
 111         if $map->{entries} && defined($map->{entries}->{$source});
 112
 113     return $map->{default} if $map->{default};
 114
 115     # identity (fallback)
 116     return $source;
 117 }
 118
 119 PVE::JSONSchema::register_standard_option('pve-targetstorage', {
 120     description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
 121     type => 'string',
 122     format => 'storagepair-list',
 123     optional => 1,
 124 });
 125
 126 #no warnings 'redefine';
 127
 128 my $nodename_cache;
 129 sub nodename {
 130     $nodename_cache //= PVE::INotify::nodename();
 131     return $nodename_cache;
 132 }
 133
 134 my $watchdog_fmt = {
 135     model => {
 136         default_key => 1,
 137         type => 'string',
 138         enum => [qw(i6300esb ib700)],
 139         description => "Watchdog type to emulate.",
 140         default => 'i6300esb',
 141         optional => 1,
 142     },
 143     action => {
 144         type => 'string',
 145         enum => [qw(reset shutdown poweroff pause debug none)],
 146         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 147         optional => 1,
 148     },
 149 };
 150 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 151
 152 my $agent_fmt = {
 153     enabled => {
 154         description => "Enable/disable Qemu GuestAgent.",
 155         type => 'boolean',
 156         default => 0,
 157         default_key => 1,
 158     },
 159     fstrim_cloned_disks => {
 160         description => "Run fstrim after moving a disk or migrating the VM.",
 161         type => 'boolean',
 162         optional => 1,
 163         default => 0
 164     },
 165     type => {
 166         description => "Select the agent type",
 167         type => 'string',
 168         default => 'virtio',
 169         optional => 1,
 170         enum => [qw(virtio isa)],
 171     },
 172 };
 173
 174 my $vga_fmt = {
 175     type => {
 176         description => "Select the VGA type.",
 177         type => 'string',
 178         default => 'std',
 179         optional => 1,
 180         default_key => 1,
 181         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 182     },
 183     memory => {
 184         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 185         type => 'integer',
 186         optional => 1,
 187         minimum => 4,
 188         maximum => 512,
 189     },
 190 };
 191
 192 my $ivshmem_fmt = {
 193     size => {
 194         type => 'integer',
 195         minimum => 1,
 196         description => "The size of the file in MB.",
 197     },
 198     name => {
 199         type => 'string',
 200         pattern => '[a-zA-Z0-9\-]+',
 201         optional => 1,
 202         format_description => 'string',
 203         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 204     },
 205 };
 206
 207 my $audio_fmt = {
 208     device => {
 209         type => 'string',
 210         enum => [qw(ich9-intel-hda intel-hda AC97)],
 211         description =>  "Configure an audio device."
 212     },
 213     driver =>  {
 214         type => 'string',
 215         enum => ['spice', 'none'],
 216         default => 'spice',
 217         optional => 1,
 218         description => "Driver backend for the audio device."
 219     },
 220 };
 221
 222 my $spice_enhancements_fmt = {
 223     foldersharing => {
 224         type => 'boolean',
 225         optional => 1,
 226         default => '0',
 227         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 228     },
 229     videostreaming =>  {
 230         type => 'string',
 231         enum => ['off', 'all', 'filter'],
 232         default => 'off',
 233         optional => 1,
 234         description => "Enable video streaming. Uses compression for detected video streams."
 235     },
 236 };
 237
 238 my $rng_fmt = {
 239     source => {
 240         type => 'string',
 241         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 242         default_key => 1,
 243         description => "The file on the host to gather entropy from. In most"
 244                      . " cases /dev/urandom should be preferred over /dev/random"
 245                      . " to avoid entropy-starvation issues on the host. Using"
 246                      . " urandom does *not* decrease security in any meaningful"
 247                      . " way, as it's still seeded from real entropy, and the"
 248                      . " bytes provided will most likely be mixed with real"
 249                      . " entropy on the guest as well. /dev/hwrng can be used"
 250                      . " to pass through a hardware RNG from the host.",
 251     },
 252     max_bytes => {
 253         type => 'integer',
 254         description => "Maximum bytes of entropy injected into the guest every"
 255                      . " 'period' milliseconds. Prefer a lower value when using"
 256                      . " /dev/random as source. Use 0 to disable limiting"
 257                      . " (potentially dangerous!).",
 258         optional => 1,
 259
 260         # default is 1 KiB/s, provides enough entropy to the guest to avoid
 261         # boot-starvation issues (e.g. systemd etc...) while allowing no chance
 262         # of overwhelming the host, provided we're reading from /dev/urandom
 263         default => 1024,
 264     },
 265     period => {
 266         type => 'integer',
 267         description => "Every 'period' milliseconds the entropy-injection quota"
 268                      . " is reset, allowing the guest to retrieve another"
 269                      . " 'max_bytes' of entropy.",
 270         optional => 1,
 271         default => 1000,
 272     },
 273 };
 274
 275 my $confdesc = {
 276     onboot => {
 277         optional => 1,
 278         type => 'boolean',
 279         description => "Specifies whether a VM will be started during system bootup.",
 280         default => 0,
 281     },
 282     autostart => {
 283         optional => 1,
 284         type => 'boolean',
 285         description => "Automatic restart after crash (currently ignored).",
 286         default => 0,
 287     },
 288     hotplug => {
 289         optional => 1,
 290         type => 'string', format => 'pve-hotplug-features',
 291         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 292         default => 'network,disk,usb',
 293     },
 294     reboot => {
 295         optional => 1,
 296         type => 'boolean',
 297         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 298         default => 1,
 299     },
 300     lock => {
 301         optional => 1,
 302         type => 'string',
 303         description => "Lock/unlock the VM.",
 304         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 305     },
 306     cpulimit => {
 307         optional => 1,
 308         type => 'number',
 309         description => "Limit of CPU usage.",
 310         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 311         minimum => 0,
 312         maximum => 128,
 313         default => 0,
 314     },
 315     cpuunits => {
 316         optional => 1,
 317         type => 'integer',
 318         description => "CPU weight for a VM.",
 319         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 320         minimum => 2,
 321         maximum => 262144,
 322         default => 1024,
 323     },
 324     memory => {
 325         optional => 1,
 326         type => 'integer',
 327         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 328         minimum => 16,
 329         default => 512,
 330     },
 331     balloon => {
 332         optional => 1,
 333         type => 'integer',
 334         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 335         minimum => 0,
 336     },
 337     shares => {
 338         optional => 1,
 339         type => 'integer',
 340         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 341         minimum => 0,
 342         maximum => 50000,
 343         default => 1000,
 344     },
 345     keyboard => {
 346         optional => 1,
 347         type => 'string',
 348         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 349                        "It should not be necessary to set it.",
 350         enum => PVE::Tools::kvmkeymaplist(),
 351         default => undef,
 352     },
 353     name => {
 354         optional => 1,
 355         type => 'string', format => 'dns-name',
 356         description => "Set a name for the VM. Only used on the configuration web interface.",
 357     },
 358     scsihw => {
 359         optional => 1,
 360         type => 'string',
 361         description => "SCSI controller model",
 362         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 363         default => 'lsi',
 364     },
 365     description => {
 366         optional => 1,
 367         type => 'string',
 368         description => "Description for the VM. Shown in the web-interface VM's summary."
 369             ." This is saved as comment inside the configuration file.",
 370         maxLength => 1024 * 8,
 371     },
 372     ostype => {
 373         optional => 1,
 374         type => 'string',
 375         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 376         description => "Specify guest operating system.",
 377         verbose_description => <<EODESC,
 378 Specify guest operating system. This is used to enable special
 379 optimization/features for specific operating systems:
 380
 381 [horizontal]
 382 other;; unspecified OS
 383 wxp;; Microsoft Windows XP
 384 w2k;; Microsoft Windows 2000
 385 w2k3;; Microsoft Windows 2003
 386 w2k8;; Microsoft Windows 2008
 387 wvista;; Microsoft Windows Vista
 388 win7;; Microsoft Windows 7
 389 win8;; Microsoft Windows 8/2012/2012r2
 390 win10;; Microsoft Windows 10/2016/2019
 391 l24;; Linux 2.4 Kernel
 392 l26;; Linux 2.6 - 5.X Kernel
 393 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 394 EODESC
 395     },
 396     boot => {
 397         optional => 1,
 398         type => 'string', format => 'pve-qm-boot',
 399         description => "Specify guest boot order. Use with 'order=', usage with"
 400                      . " no key or 'legacy=' is deprecated.",
 401     },
 402     bootdisk => {
 403         optional => 1,
 404         type => 'string', format => 'pve-qm-bootdisk',
 405         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 406         pattern => '(ide|sata|scsi|virtio)\d+',
 407     },
 408     smp => {
 409         optional => 1,
 410         type => 'integer',
 411         description => "The number of CPUs. Please use option -sockets instead.",
 412         minimum => 1,
 413         default => 1,
 414     },
 415     sockets => {
 416         optional => 1,
 417         type => 'integer',
 418         description => "The number of CPU sockets.",
 419         minimum => 1,
 420         default => 1,
 421     },
 422     cores => {
 423         optional => 1,
 424         type => 'integer',
 425         description => "The number of cores per socket.",
 426         minimum => 1,
 427         default => 1,
 428     },
 429     numa => {
 430         optional => 1,
 431         type => 'boolean',
 432         description => "Enable/disable NUMA.",
 433         default => 0,
 434     },
 435     hugepages => {
 436         optional => 1,
 437         type => 'string',
 438         description => "Enable/disable hugepages memory.",
 439         enum => [qw(any 2 1024)],
 440     },
 441     keephugepages => {
 442         optional => 1,
 443         type => 'boolean',
 444         default => 0,
 445         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 446             ." after VM shutdown and can be used for subsequent starts.",
 447     },
 448     vcpus => {
 449         optional => 1,
 450         type => 'integer',
 451         description => "Number of hotplugged vcpus.",
 452         minimum => 1,
 453         default => 0,
 454     },
 455     acpi => {
 456         optional => 1,
 457         type => 'boolean',
 458         description => "Enable/disable ACPI.",
 459         default => 1,
 460     },
 461     agent => {
 462         optional => 1,
 463         description => "Enable/disable Qemu GuestAgent and its properties.",
 464         type => 'string',
 465         format => $agent_fmt,
 466     },
 467     kvm => {
 468         optional => 1,
 469         type => 'boolean',
 470         description => "Enable/disable KVM hardware virtualization.",
 471         default => 1,
 472     },
 473     tdf => {
 474         optional => 1,
 475         type => 'boolean',
 476         description => "Enable/disable time drift fix.",
 477         default => 0,
 478     },
 479     localtime => {
 480         optional => 1,
 481         type => 'boolean',
 482         description => "Set the real time clock to local time. This is enabled by default if ostype"
 483             ." indicates a Microsoft OS.",
 484     },
 485     freeze => {
 486         optional => 1,
 487         type => 'boolean',
 488         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 489     },
 490     vga => {
 491         optional => 1,
 492         type => 'string', format => $vga_fmt,
 493         description => "Configure the VGA hardware.",
 494         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 495             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 496             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 497             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 498             ." display server. For win* OS you can select how many independent displays you want,"
 499             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 500             ." using a serial device as terminal.",
 501     },
 502     watchdog => {
 503         optional => 1,
 504         type => 'string', format => 'pve-qm-watchdog',
 505         description => "Create a virtual hardware watchdog device.",
 506         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 507             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 508             ." else the watchdog will reset the guest (or execute the respective action specified)",
 509     },
 510     startdate => {
 511         optional => 1,
 512         type => 'string',
 513         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 514         description => "Set the initial date of the real time clock. Valid format for date are:"
 515             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 516         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 517         default => 'now',
 518     },
 519     startup =>  get_standard_option('pve-startup-order'),
 520     template => {
 521         optional => 1,
 522         type => 'boolean',
 523         description => "Enable/disable Template.",
 524         default => 0,
 525     },
 526     args => {
 527         optional => 1,
 528         type => 'string',
 529         description => "Arbitrary arguments passed to kvm.",
 530         verbose_description => <<EODESCR,
 531 Arbitrary arguments passed to kvm, for example:
 532
 533 args: -no-reboot -no-hpet
 534
 535 NOTE: this option is for experts only.
 536 EODESCR
 537     },
 538     tablet => {
 539         optional => 1,
 540         type => 'boolean',
 541         default => 1,
 542         description => "Enable/disable the USB tablet device.",
 543         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 544             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 545             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 546             ." may consider disabling this to save some context switches. This is turned off by"
 547             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 548     },
 549     migrate_speed => {
 550         optional => 1,
 551         type => 'integer',
 552         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 553         minimum => 0,
 554         default => 0,
 555     },
 556     migrate_downtime => {
 557         optional => 1,
 558         type => 'number',
 559         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 560         minimum => 0,
 561         default => 0.1,
 562     },
 563     cdrom => {
 564         optional => 1,
 565         type => 'string', format => 'pve-qm-ide',
 566         typetext => '<volume>',
 567         description => "This is an alias for option -ide2",
 568     },
 569     cpu => {
 570         optional => 1,
 571         description => "Emulated CPU type.",
 572         type => 'string',
 573         format => 'pve-vm-cpu-conf',
 574     },
 575     parent => get_standard_option('pve-snapshot-name', {
 576         optional => 1,
 577         description => "Parent snapshot name. This is used internally, and should not be modified.",
 578     }),
 579     snaptime => {
 580         optional => 1,
 581         description => "Timestamp for snapshots.",
 582         type => 'integer',
 583         minimum => 0,
 584     },
 585     vmstate => {
 586         optional => 1,
 587         type => 'string', format => 'pve-volume-id',
 588         description => "Reference to a volume which stores the VM state. This is used internally"
 589             ." for snapshots.",
 590     },
 591     vmstatestorage => get_standard_option('pve-storage-id', {
 592         description => "Default storage for VM state volumes/files.",
 593         optional => 1,
 594     }),
 595     runningmachine => get_standard_option('pve-qemu-machine', {
 596         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 597             ." for snapshots.",
 598     }),
 599     runningcpu => {
 600         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 601             ." internally for snapshots.",
 602         optional => 1,
 603         type => 'string',
 604         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 605         format_description => 'QEMU -cpu parameter'
 606     },
 607     machine => get_standard_option('pve-qemu-machine'),
 608     arch => {
 609         description => "Virtual processor architecture. Defaults to the host.",
 610         optional => 1,
 611         type => 'string',
 612         enum => [qw(x86_64 aarch64)],
 613     },
 614     smbios1 => {
 615         description => "Specify SMBIOS type 1 fields.",
 616         type => 'string', format => 'pve-qm-smbios1',
 617         maxLength => 512,
 618         optional => 1,
 619     },
 620     protection => {
 621         optional => 1,
 622         type => 'boolean',
 623         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 624             ." remove disk operations.",
 625         default => 0,
 626     },
 627     bios => {
 628         optional => 1,
 629         type => 'string',
 630         enum => [ qw(seabios ovmf) ],
 631         description => "Select BIOS implementation.",
 632         default => 'seabios',
 633     },
 634     vmgenid => {
 635         type => 'string',
 636         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 637         format_description => 'UUID',
 638         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 639             ." to disable explicitly.",
 640         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 641             ." value identifier to the guest OS. This allows to notify the guest operating system"
 642             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 643             ." execution or creation from a template). The guest operating system notices the"
 644             ." change, and is then able to react as appropriate by marking its copies of"
 645             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 646             ."Note that auto-creation only works when done through API/CLI create or update methods"
 647             .", but not when manually editing the config file.",
 648         default => "1 (autogenerated)",
 649         optional => 1,
 650     },
 651     hookscript => {
 652         type => 'string',
 653         format => 'pve-volume-id',
 654         optional => 1,
 655         description => "Script that will be executed during various steps in the vms lifetime.",
 656     },
 657     ivshmem => {
 658         type => 'string',
 659         format => $ivshmem_fmt,
 660         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 661             ." the host.",
 662         optional => 1,
 663     },
 664     audio0 => {
 665         type => 'string',
 666         format => $audio_fmt,
 667         description => "Configure a audio device, useful in combination with QXL/Spice.",
 668         optional => 1
 669     },
 670     spice_enhancements => {
 671         type => 'string',
 672         format => $spice_enhancements_fmt,
 673         description => "Configure additional enhancements for SPICE.",
 674         optional => 1
 675     },
 676     tags => {
 677         type => 'string', format => 'pve-tag-list',
 678         description => 'Tags of the VM. This is only meta information.',
 679         optional => 1,
 680     },
 681     rng0 => {
 682         type => 'string',
 683         format => $rng_fmt,
 684         description => "Configure a VirtIO-based Random Number Generator.",
 685         optional => 1,
 686     },
 687 };
 688
 689 my $cicustom_fmt = {
 690     meta => {
 691         type => 'string',
 692         optional => 1,
 693         description => 'Specify a custom file containing all meta data passed to the VM via"
 694             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 695         format => 'pve-volume-id',
 696         format_description => 'volume',
 697     },
 698     network => {
 699         type => 'string',
 700         optional => 1,
 701         description => 'Specify a custom file containing all network data passed to the VM via'
 702             .' cloud-init.',
 703         format => 'pve-volume-id',
 704         format_description => 'volume',
 705     },
 706     user => {
 707         type => 'string',
 708         optional => 1,
 709         description => 'Specify a custom file containing all user data passed to the VM via'
 710             .' cloud-init.',
 711         format => 'pve-volume-id',
 712         format_description => 'volume',
 713     },
 714 };
 715 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 716
 717 my $confdesc_cloudinit = {
 718     citype => {
 719         optional => 1,
 720         type => 'string',
 721         description => 'Specifies the cloud-init configuration format. The default depends on the'
 722             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 723             .' and `configdrive2` for windows.',
 724         enum => ['configdrive2', 'nocloud', 'opennebula'],
 725     },
 726     ciuser => {
 727         optional => 1,
 728         type => 'string',
 729         description => "cloud-init: User name to change ssh keys and password for instead of the"
 730             ." image's configured default user.",
 731     },
 732     cipassword => {
 733         optional => 1,
 734         type => 'string',
 735         description => 'cloud-init: Password to assign the user. Using this is generally not'
 736             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 737             .' support hashed passwords.',
 738     },
 739     cicustom => {
 740         optional => 1,
 741         type => 'string',
 742         description => 'cloud-init: Specify custom files to replace the automatically generated'
 743             .' ones at start.',
 744         format => 'pve-qm-cicustom',
 745     },
 746     searchdomain => {
 747         optional => 1,
 748         type => 'string',
 749         description => "cloud-init: Sets DNS search domains for a container. Create will'
 750             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 751             .' are set.",
 752     },
 753     nameserver => {
 754         optional => 1,
 755         type => 'string', format => 'address-list',
 756         description => "cloud-init: Sets DNS server IP address for a container. Create will'
 757             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 758             .' are set.",
 759     },
 760     sshkeys => {
 761         optional => 1,
 762         type => 'string',
 763         format => 'urlencoded',
 764         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 765     },
 766 };
 767
 768 # what about other qemu settings ?
 769 #cpu => 'string',
 770 #machine => 'string',
 771 #fda => 'file',
 772 #fdb => 'file',
 773 #mtdblock => 'file',
 774 #sd => 'file',
 775 #pflash => 'file',
 776 #snapshot => 'bool',
 777 #bootp => 'file',
 778 ##tftp => 'dir',
 779 ##smb => 'dir',
 780 #kernel => 'file',
 781 #append => 'string',
 782 #initrd => 'file',
 783 ##soundhw => 'string',
 784
 785 while (my ($k, $v) = each %$confdesc) {
 786     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 787 }
 788
 789 my $MAX_USB_DEVICES = 5;
 790 my $MAX_NETS = 32;
 791 my $MAX_SERIAL_PORTS = 4;
 792 my $MAX_PARALLEL_PORTS = 3;
 793 my $MAX_NUMA = 8;
 794
 795 my $numa_fmt = {
 796     cpus => {
 797         type => "string",
 798         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 799         description => "CPUs accessing this NUMA node.",
 800         format_description => "id[-id];...",
 801     },
 802     memory => {
 803         type => "number",
 804         description => "Amount of memory this NUMA node provides.",
 805         optional => 1,
 806     },
 807     hostnodes => {
 808         type => "string",
 809         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 810         description => "Host NUMA nodes to use.",
 811         format_description => "id[-id];...",
 812         optional => 1,
 813     },
 814     policy => {
 815         type => 'string',
 816         enum => [qw(preferred bind interleave)],
 817         description => "NUMA allocation policy.",
 818         optional => 1,
 819     },
 820 };
 821 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 822 my $numadesc = {
 823     optional => 1,
 824     type => 'string', format => $numa_fmt,
 825     description => "NUMA topology.",
 826 };
 827 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 828
 829 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 830     $confdesc->{"numa$i"} = $numadesc;
 831 }
 832
 833 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 834                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 835                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 836 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 837
 838 my $net_fmt_bridge_descr = <<__EOD__;
 839 Bridge to attach the network device to. The Proxmox VE standard bridge
 840 is called 'vmbr0'.
 841
 842 If you do not specify a bridge, we create a kvm user (NATed) network
 843 device, which provides DHCP and DNS services. The following addresses
 844 are used:
 845
 846  10.0.2.2   Gateway
 847  10.0.2.3   DNS Server
 848  10.0.2.4   SMB Server
 849
 850 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 851 __EOD__
 852
 853 my $net_fmt = {
 854     macaddr  => get_standard_option('mac-addr', {
 855         description => "MAC address. That address must be unique withing your network. This is"
 856             ." automatically generated if not specified.",
 857     }),
 858     model => {
 859         type => 'string',
 860         description => "Network Card Model. The 'virtio' model provides the best performance with"
 861             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 862             ." best to use 'e1000'.",
 863         enum => $nic_model_list,
 864         default_key => 1,
 865     },
 866     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 867     bridge => {
 868         type => 'string',
 869         description => $net_fmt_bridge_descr,
 870         format_description => 'bridge',
 871         pattern => '[-_.\w\d]+',
 872         optional => 1,
 873     },
 874     queues => {
 875         type => 'integer',
 876         minimum => 0, maximum => 16,
 877         description => 'Number of packet queues to be used on the device.',
 878         optional => 1,
 879     },
 880     rate => {
 881         type => 'number',
 882         minimum => 0,
 883         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 884         optional => 1,
 885     },
 886     tag => {
 887         type => 'integer',
 888         minimum => 1, maximum => 4094,
 889         description => 'VLAN tag to apply to packets on this interface.',
 890         optional => 1,
 891     },
 892     trunks => {
 893         type => 'string',
 894         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 895         description => 'VLAN trunks to pass through this interface.',
 896         format_description => 'vlanid[;vlanid...]',
 897         optional => 1,
 898     },
 899     firewall => {
 900         type => 'boolean',
 901         description => 'Whether this interface should be protected by the firewall.',
 902         optional => 1,
 903     },
 904     link_down => {
 905         type => 'boolean',
 906         description => 'Whether this interface should be disconnected (like pulling the plug).',
 907         optional => 1,
 908     },
 909     mtu => {
 910         type => 'integer',
 911         minimum => 1, maximum => 65520,
 912         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 913         optional => 1,
 914     },
 915 };
 916
 917 my $netdesc = {
 918     optional => 1,
 919     type => 'string', format => $net_fmt,
 920     description => "Specify network devices.",
 921 };
 922
 923 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 924
 925 my $ipconfig_fmt = {
 926     ip => {
 927         type => 'string',
 928         format => 'pve-ipv4-config',
 929         format_description => 'IPv4Format/CIDR',
 930         description => 'IPv4 address in CIDR format.',
 931         optional => 1,
 932         default => 'dhcp',
 933     },
 934     gw => {
 935         type => 'string',
 936         format => 'ipv4',
 937         format_description => 'GatewayIPv4',
 938         description => 'Default gateway for IPv4 traffic.',
 939         optional => 1,
 940         requires => 'ip',
 941     },
 942     ip6 => {
 943         type => 'string',
 944         format => 'pve-ipv6-config',
 945         format_description => 'IPv6Format/CIDR',
 946         description => 'IPv6 address in CIDR format.',
 947         optional => 1,
 948         default => 'dhcp',
 949     },
 950     gw6 => {
 951         type => 'string',
 952         format => 'ipv6',
 953         format_description => 'GatewayIPv6',
 954         description => 'Default gateway for IPv6 traffic.',
 955         optional => 1,
 956         requires => 'ip6',
 957     },
 958 };
 959 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 960 my $ipconfigdesc = {
 961     optional => 1,
 962     type => 'string', format => 'pve-qm-ipconfig',
 963     description => <<'EODESCR',
 964 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 965
 966 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 967
 968 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
 969 gateway should be provided.
 970 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
 971 cloud-init 19.4 or newer.
 972
 973 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
 974 dhcp on IPv4.
 975 EODESCR
 976 };
 977 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 978
 979 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 980     $confdesc->{"net$i"} = $netdesc;
 981     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 982 }
 983
 984 foreach my $key (keys %$confdesc_cloudinit) {
 985     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 986 }
 987
 988 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 989 sub verify_volume_id_or_qm_path {
 990     my ($volid, $noerr) = @_;
 991
 992     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 993         return $volid;
 994     }
 995
 996     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 997     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 998     if ($@) {
 999         return if $noerr;
1000         die $@;
1001     }
1002     return $volid;
1003 }
1004
1005 my $usb_fmt = {
1006     host => {
1007         default_key => 1,
1008         type => 'string', format => 'pve-qm-usb-device',
1009         format_description => 'HOSTUSBDEVICE|spice',
1010         description => <<EODESCR,
1011 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1012
1013  'bus-port(.port)*' (decimal numbers) or
1014  'vendor_id:product_id' (hexadeciaml numbers) or
1015  'spice'
1016
1017 You can use the 'lsusb -t' command to list existing usb devices.
1018
1019 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1020 machines - use with special care.
1021
1022 The value 'spice' can be used to add a usb redirection devices for spice.
1023 EODESCR
1024     },
1025     usb3 => {
1026         optional => 1,
1027         type => 'boolean',
1028         description => "Specifies whether if given host option is a USB3 device or port.",
1029         default => 0,
1030     },
1031 };
1032
1033 my $usbdesc = {
1034     optional => 1,
1035     type => 'string', format => $usb_fmt,
1036     description => "Configure an USB device (n is 0 to 4).",
1037 };
1038 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1039
1040 my $serialdesc = {
1041         optional => 1,
1042         type => 'string',
1043         pattern => '(/dev/.+|socket)',
1044         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1045         verbose_description =>  <<EODESCR,
1046 Create a serial device inside the VM (n is 0 to 3), and pass through a
1047 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1048 host side (use 'qm terminal' to open a terminal connection).
1049
1050 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1051 use with special care.
1052
1053 CAUTION: Experimental! User reported problems with this option.
1054 EODESCR
1055 };
1056
1057 my $paralleldesc= {
1058         optional => 1,
1059         type => 'string',
1060         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1061         description =>  "Map host parallel devices (n is 0 to 2).",
1062         verbose_description =>  <<EODESCR,
1063 Map host parallel devices (n is 0 to 2).
1064
1065 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1066 machines - use with special care.
1067
1068 CAUTION: Experimental! User reported problems with this option.
1069 EODESCR
1070 };
1071
1072 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1073     $confdesc->{"parallel$i"} = $paralleldesc;
1074 }
1075
1076 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1077     $confdesc->{"serial$i"} = $serialdesc;
1078 }
1079
1080 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1081     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1082 }
1083
1084 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1085     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1086 }
1087
1088 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1089     $confdesc->{"usb$i"} = $usbdesc;
1090 }
1091
1092 my $boot_fmt = {
1093     legacy => {
1094         optional => 1,
1095         default_key => 1,
1096         type => 'string',
1097         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1098                      . " Deprecated, use 'order=' instead.",
1099         pattern => '[acdn]{1,4}',
1100         format_description => "[acdn]{1,4}",
1101
1102         # note: this is also the fallback if boot: is not given at all
1103         default => 'cdn',
1104     },
1105     order => {
1106         optional => 1,
1107         type => 'string',
1108         format => 'pve-qm-bootdev-list',
1109         format_description => "device[;device...]",
1110         description => <<EODESC,
1111 The guest will attempt to boot from devices in the order they appear here.
1112
1113 Disks, optical drives and passed-through storage USB devices will be directly
1114 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1115 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1116
1117 Note that only devices in this list will be marked as bootable and thus loaded
1118 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1119 (e.g. software-raid), you need to specify all of them here.
1120
1121 Overrides the deprecated 'legacy=[acdn]*' value when given.
1122 EODESC
1123     },
1124 };
1125 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1126
1127 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1128 sub verify_bootdev {
1129     my ($dev, $noerr) = @_;
1130
1131     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && $dev !~ m/^efidisk/;
1132
1133     my $check = sub {
1134         my ($base) = @_;
1135         return 0 if $dev !~ m/^$base\d+$/;
1136         return 0 if !$confdesc->{$dev};
1137         return 1;
1138     };
1139
1140     return $dev if $check->("net");
1141     return $dev if $check->("usb");
1142     return $dev if $check->("hostpci");
1143
1144     return if $noerr;
1145     die "invalid boot device '$dev'\n";
1146 }
1147
1148 sub print_bootorder {
1149     my ($devs) = @_;
1150     return "" if !@$devs;
1151     my $data = { order => join(';', @$devs) };
1152     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1153 }
1154
1155 my $kvm_api_version = 0;
1156
1157 sub kvm_version {
1158     return $kvm_api_version if $kvm_api_version;
1159
1160     open my $fh, '<', '/dev/kvm' or return;
1161
1162     # 0xae00 => KVM_GET_API_VERSION
1163     $kvm_api_version = ioctl($fh, 0xae00, 0);
1164     close($fh);
1165
1166     return $kvm_api_version;
1167 }
1168
1169 my $kvm_user_version = {};
1170 my $kvm_mtime = {};
1171
1172 sub kvm_user_version {
1173     my ($binary) = @_;
1174
1175     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1176     my $st = stat($binary);
1177
1178     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1179     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1180         $cachedmtime == $st->mtime;
1181
1182     $kvm_user_version->{$binary} = 'unknown';
1183     $kvm_mtime->{$binary} = $st->mtime;
1184
1185     my $code = sub {
1186         my $line = shift;
1187         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1188             $kvm_user_version->{$binary} = $2;
1189         }
1190     };
1191
1192     eval { run_command([$binary, '--version'], outfunc => $code); };
1193     warn $@ if $@;
1194
1195     return $kvm_user_version->{$binary};
1196
1197 }
1198 my sub extract_version {
1199     my ($machine_type, $version) = @_;
1200     $version = kvm_user_version() if !defined($version);
1201     PVE::QemuServer::Machine::extract_version($machine_type, $version)
1202 }
1203
1204 sub kernel_has_vhost_net {
1205     return -c '/dev/vhost-net';
1206 }
1207
1208 sub option_exists {
1209     my $key = shift;
1210     return defined($confdesc->{$key});
1211 }
1212
1213 my $cdrom_path;
1214 sub get_cdrom_path {
1215
1216     return  $cdrom_path if $cdrom_path;
1217
1218     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1219     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1220     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1221 }
1222
1223 sub get_iso_path {
1224     my ($storecfg, $vmid, $cdrom) = @_;
1225
1226     if ($cdrom eq 'cdrom') {
1227         return get_cdrom_path();
1228     } elsif ($cdrom eq 'none') {
1229         return '';
1230     } elsif ($cdrom =~ m|^/|) {
1231         return $cdrom;
1232     } else {
1233         return PVE::Storage::path($storecfg, $cdrom);
1234     }
1235 }
1236
1237 # try to convert old style file names to volume IDs
1238 sub filename_to_volume_id {
1239     my ($vmid, $file, $media) = @_;
1240
1241      if (!($file eq 'none' || $file eq 'cdrom' ||
1242           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1243
1244         return if $file =~ m|/|;
1245
1246         if ($media && $media eq 'cdrom') {
1247             $file = "local:iso/$file";
1248         } else {
1249             $file = "local:$vmid/$file";
1250         }
1251     }
1252
1253     return $file;
1254 }
1255
1256 sub verify_media_type {
1257     my ($opt, $vtype, $media) = @_;
1258
1259     return if !$media;
1260
1261     my $etype;
1262     if ($media eq 'disk') {
1263         $etype = 'images';
1264     } elsif ($media eq 'cdrom') {
1265         $etype = 'iso';
1266     } else {
1267         die "internal error";
1268     }
1269
1270     return if ($vtype eq $etype);
1271
1272     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1273 }
1274
1275 sub cleanup_drive_path {
1276     my ($opt, $storecfg, $drive) = @_;
1277
1278     # try to convert filesystem paths to volume IDs
1279
1280     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1281         ($drive->{file} !~ m|^/dev/.+|) &&
1282         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1283         ($drive->{file} !~ m/^\d+$/)) {
1284         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1285         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1286             if !$vtype;
1287         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1288         verify_media_type($opt, $vtype, $drive->{media});
1289         $drive->{file} = $volid;
1290     }
1291
1292     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1293 }
1294
1295 sub parse_hotplug_features {
1296     my ($data) = @_;
1297
1298     my $res = {};
1299
1300     return $res if $data eq '0';
1301
1302     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1303
1304     foreach my $feature (PVE::Tools::split_list($data)) {
1305         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1306             $res->{$1} = 1;
1307         } else {
1308             die "invalid hotplug feature '$feature'\n";
1309         }
1310     }
1311     return $res;
1312 }
1313
1314 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1315 sub pve_verify_hotplug_features {
1316     my ($value, $noerr) = @_;
1317
1318     return $value if parse_hotplug_features($value);
1319
1320     return if $noerr;
1321
1322     die "unable to parse hotplug option\n";
1323 }
1324
1325 sub scsi_inquiry {
1326     my($fh, $noerr) = @_;
1327
1328     my $SG_IO = 0x2285;
1329     my $SG_GET_VERSION_NUM = 0x2282;
1330
1331     my $versionbuf = "\x00" x 8;
1332     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1333     if (!$ret) {
1334         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1335         return;
1336     }
1337     my $version = unpack("I", $versionbuf);
1338     if ($version < 30000) {
1339         die "scsi generic interface too old\n"  if !$noerr;
1340         return;
1341     }
1342
1343     my $buf = "\x00" x 36;
1344     my $sensebuf = "\x00" x 8;
1345     my $cmd = pack("C x3 C x1", 0x12, 36);
1346
1347     # see /usr/include/scsi/sg.h
1348     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1349
1350     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1351                       length($sensebuf), 0, length($buf), $buf,
1352                       $cmd, $sensebuf, 6000);
1353
1354     $ret = ioctl($fh, $SG_IO, $packet);
1355     if (!$ret) {
1356         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1357         return;
1358     }
1359
1360     my @res = unpack($sg_io_hdr_t, $packet);
1361     if ($res[17] || $res[18]) {
1362         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1363         return;
1364     }
1365
1366     my $res = {};
1367     (my $byte0, my $byte1, $res->{vendor},
1368      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1369
1370     $res->{removable} = $byte1 & 128 ? 1 : 0;
1371     $res->{type} = $byte0 & 31;
1372
1373     return $res;
1374 }
1375
1376 sub path_is_scsi {
1377     my ($path) = @_;
1378
1379     my $fh = IO::File->new("+<$path") || return;
1380     my $res = scsi_inquiry($fh, 1);
1381     close($fh);
1382
1383     return $res;
1384 }
1385
1386 sub print_tabletdevice_full {
1387     my ($conf, $arch) = @_;
1388
1389     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1390
1391     # we use uhci for old VMs because tablet driver was buggy in older qemu
1392     my $usbbus;
1393     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1394         $usbbus = 'ehci';
1395     } else {
1396         $usbbus = 'uhci';
1397     }
1398
1399     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1400 }
1401
1402 sub print_keyboarddevice_full {
1403     my ($conf, $arch, $machine) = @_;
1404
1405     return if $arch ne 'aarch64';
1406
1407     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1408 }
1409
1410 my sub get_drive_id {
1411     my ($drive) = @_;
1412     return "$drive->{interface}$drive->{index}";
1413 }
1414
1415 sub print_drivedevice_full {
1416     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1417
1418     my $device = '';
1419     my $maxdev = 0;
1420
1421     my $drive_id = get_drive_id($drive);
1422     if ($drive->{interface} eq 'virtio') {
1423         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1424         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1425         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1426     } elsif ($drive->{interface} eq 'scsi') {
1427
1428         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1429         my $unit = $drive->{index} % $maxdev;
1430         my $devicetype = 'hd';
1431         my $path = '';
1432         if (drive_is_cdrom($drive)) {
1433             $devicetype = 'cd';
1434         } else {
1435             if ($drive->{file} =~ m|^/|) {
1436                 $path = $drive->{file};
1437                 if (my $info = path_is_scsi($path)) {
1438                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1439                         $devicetype = 'block';
1440                     } elsif ($info->{type} == 1) { # tape
1441                         $devicetype = 'generic';
1442                     }
1443                 }
1444             } else {
1445                  $path = PVE::Storage::path($storecfg, $drive->{file});
1446             }
1447
1448             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1449             my $version = extract_version($machine_type, kvm_user_version());
1450             if ($path =~ m/^iscsi\:\/\// &&
1451                !min_version($version, 4, 1)) {
1452                 $devicetype = 'generic';
1453             }
1454         }
1455
1456         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1457             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1458         } else {
1459             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1460                 .",lun=$drive->{index}";
1461         }
1462         $device .= ",drive=drive-$drive_id,id=$drive_id";
1463
1464         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1465             $device .= ",rotation_rate=1";
1466         }
1467         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1468
1469     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1470         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1471         my $controller = int($drive->{index} / $maxdev);
1472         my $unit = $drive->{index} % $maxdev;
1473         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1474
1475         $device = "ide-$devicetype";
1476         if ($drive->{interface} eq 'ide') {
1477             $device .= ",bus=ide.$controller,unit=$unit";
1478         } else {
1479             $device .= ",bus=ahci$controller.$unit";
1480         }
1481         $device .= ",drive=drive-$drive_id,id=$drive_id";
1482
1483         if ($devicetype eq 'hd') {
1484             if (my $model = $drive->{model}) {
1485                 $model = URI::Escape::uri_unescape($model);
1486                 $device .= ",model=$model";
1487             }
1488             if ($drive->{ssd}) {
1489                 $device .= ",rotation_rate=1";
1490             }
1491         }
1492         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1493     } elsif ($drive->{interface} eq 'usb') {
1494         die "implement me";
1495         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1496     } else {
1497         die "unsupported interface type";
1498     }
1499
1500     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1501
1502     if (my $serial = $drive->{serial}) {
1503         $serial = URI::Escape::uri_unescape($serial);
1504         $device .= ",serial=$serial";
1505     }
1506
1507
1508     return $device;
1509 }
1510
1511 sub get_initiator_name {
1512     my $initiator;
1513
1514     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1515     while (defined(my $line = <$fh>)) {
1516         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1517         $initiator = $1;
1518         last;
1519     }
1520     $fh->close();
1521
1522     return $initiator;
1523 }
1524
1525 sub print_drive_commandline_full {
1526     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1527
1528     my $path;
1529     my $volid = $drive->{file};
1530     my $format = $drive->{format};
1531     my $drive_id = get_drive_id($drive);
1532
1533     if (drive_is_cdrom($drive)) {
1534         $path = get_iso_path($storecfg, $vmid, $volid);
1535         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1536     } else {
1537         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1538         if ($storeid) {
1539             $path = PVE::Storage::path($storecfg, $volid);
1540             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1541             $format //= qemu_img_format($scfg, $volname);
1542         } else {
1543             $path = $volid;
1544             $format //= "raw";
1545         }
1546    }
1547
1548    my $is_rbd = $path =~ m/^rbd:/;
1549
1550     my $opts = '';
1551     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1552     foreach my $o (@qemu_drive_options) {
1553         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1554     }
1555
1556     # snapshot only accepts on|off
1557     if (defined($drive->{snapshot})) {
1558         my $v = $drive->{snapshot} ? 'on' : 'off';
1559         $opts .= ",snapshot=$v";
1560     }
1561
1562     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1563         my ($dir, $qmpname) = @$type;
1564         if (my $v = $drive->{"mbps$dir"}) {
1565             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1566         }
1567         if (my $v = $drive->{"mbps${dir}_max"}) {
1568             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1569         }
1570         if (my $v = $drive->{"bps${dir}_max_length"}) {
1571             $opts .= ",throttling.bps$qmpname-max-length=$v";
1572         }
1573         if (my $v = $drive->{"iops${dir}"}) {
1574             $opts .= ",throttling.iops$qmpname=$v";
1575         }
1576         if (my $v = $drive->{"iops${dir}_max"}) {
1577             $opts .= ",throttling.iops$qmpname-max=$v";
1578         }
1579         if (my $v = $drive->{"iops${dir}_max_length"}) {
1580             $opts .= ",throttling.iops$qmpname-max-length=$v";
1581         }
1582     }
1583
1584     if ($pbs_name) {
1585         $format = "rbd" if $is_rbd;
1586         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1587             if !$format;
1588         $opts .= ",format=alloc-track,file.driver=$format";
1589     } elsif ($format) {
1590         $opts .= ",format=$format";
1591     }
1592
1593     my $cache_direct = 0;
1594
1595     if (my $cache = $drive->{cache}) {
1596         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1597     } elsif (!drive_is_cdrom($drive)) {
1598         $opts .= ",cache=none";
1599         $cache_direct = 1;
1600     }
1601
1602     if (!$drive->{aio}) {
1603         if ($io_uring) {
1604             # io_uring supports all cache modes
1605             $opts .= ",aio=io_uring";
1606         } else {
1607             # aio native works only with O_DIRECT
1608             if($cache_direct) {
1609                 $opts .= ",aio=native";
1610             } else {
1611                 $opts .= ",aio=threads";
1612             }
1613         }
1614     }
1615
1616     if (!drive_is_cdrom($drive)) {
1617         my $detectzeroes;
1618         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1619             $detectzeroes = 'off';
1620         } elsif ($drive->{discard}) {
1621             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1622         } else {
1623             # This used to be our default with discard not being specified:
1624             $detectzeroes = 'on';
1625         }
1626
1627         # note: 'detect-zeroes' works per blockdev and we want it to persist
1628         # after the alloc-track is removed, so put it on 'file' directly
1629         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1630         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1631     }
1632
1633     if ($pbs_name) {
1634         $opts .= ",backing=$pbs_name";
1635         $opts .= ",auto-remove=on";
1636     }
1637
1638     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1639     my $file_param = "file";
1640     if ($pbs_name) {
1641         # non-rbd drivers require the underlying file to be a seperate block
1642         # node, so add a second .file indirection
1643         $file_param .= ".file" if !$is_rbd;
1644         $file_param .= ".filename";
1645     }
1646     my $pathinfo = $path ? "$file_param=$path," : '';
1647
1648     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1649 }
1650
1651 sub print_pbs_blockdev {
1652     my ($pbs_conf, $pbs_name) = @_;
1653     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1654     $blockdev .= ",repository=$pbs_conf->{repository}";
1655     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1656     $blockdev .= ",archive=$pbs_conf->{archive}";
1657     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1658     return $blockdev;
1659 }
1660
1661 sub print_netdevice_full {
1662     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1663
1664     my $device = $net->{model};
1665     if ($net->{model} eq 'virtio') {
1666          $device = 'virtio-net-pci';
1667      };
1668
1669     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1670     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1671     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1672         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1673         # and out of each queue plus one config interrupt and control vector queue
1674         my $vectors = $net->{queues} * 2 + 2;
1675         $tmpstr .= ",vectors=$vectors,mq=on";
1676     }
1677     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1678
1679     if (my $mtu = $net->{mtu}) {
1680         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1681             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1682             if ($mtu == 1) {
1683                  $mtu = $bridge_mtu;
1684             } elsif ($mtu < 576) {
1685                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1686             } elsif ($mtu > $bridge_mtu) {
1687                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1688             }
1689             $tmpstr .= ",host_mtu=$mtu";
1690         } else {
1691             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1692         }
1693     }
1694
1695     if ($use_old_bios_files) {
1696         my $romfile;
1697         if ($device eq 'virtio-net-pci') {
1698             $romfile = 'pxe-virtio.rom';
1699         } elsif ($device eq 'e1000') {
1700             $romfile = 'pxe-e1000.rom';
1701         } elsif ($device eq 'ne2k') {
1702             $romfile = 'pxe-ne2k_pci.rom';
1703         } elsif ($device eq 'pcnet') {
1704             $romfile = 'pxe-pcnet.rom';
1705         } elsif ($device eq 'rtl8139') {
1706             $romfile = 'pxe-rtl8139.rom';
1707         }
1708         $tmpstr .= ",romfile=$romfile" if $romfile;
1709     }
1710
1711     return $tmpstr;
1712 }
1713
1714 sub print_netdev_full {
1715     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1716
1717     my $i = '';
1718     if ($netid =~ m/^net(\d+)$/) {
1719         $i = int($1);
1720     }
1721
1722     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1723
1724     my $ifname = "tap${vmid}i$i";
1725
1726     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1727     die "interface name '$ifname' is too long (max 15 character)\n"
1728         if length($ifname) >= 16;
1729
1730     my $vhostparam = '';
1731     if (is_native($arch)) {
1732         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1733     }
1734
1735     my $vmname = $conf->{name} || "vm$vmid";
1736
1737     my $netdev = "";
1738     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1739
1740     if ($net->{bridge}) {
1741         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1742             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1743     } else {
1744         $netdev = "type=user,id=$netid,hostname=$vmname";
1745     }
1746
1747     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1748
1749     return $netdev;
1750 }
1751
1752 my $vga_map = {
1753     'cirrus' => 'cirrus-vga',
1754     'std' => 'VGA',
1755     'vmware' => 'vmware-svga',
1756     'virtio' => 'virtio-vga',
1757 };
1758
1759 sub print_vga_device {
1760     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1761
1762     my $type = $vga_map->{$vga->{type}};
1763     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1764         $type = 'virtio-gpu';
1765     }
1766     my $vgamem_mb = $vga->{memory};
1767
1768     my $max_outputs = '';
1769     if ($qxlnum) {
1770         $type = $id ? 'qxl' : 'qxl-vga';
1771
1772         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1773             # set max outputs so linux can have up to 4 qxl displays with one device
1774             if (min_version($machine_version, 4, 1)) {
1775                 $max_outputs = ",max_outputs=4";
1776             }
1777         }
1778     }
1779
1780     die "no devicetype for $vga->{type}\n" if !$type;
1781
1782     my $memory = "";
1783     if ($vgamem_mb) {
1784         if ($vga->{type} eq 'virtio') {
1785             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1786             $memory = ",max_hostmem=$bytes";
1787         } elsif ($qxlnum) {
1788             # from https://www.spice-space.org/multiple-monitors.html
1789             $memory = ",vgamem_mb=$vga->{memory}";
1790             my $ram = $vgamem_mb * 4;
1791             my $vram = $vgamem_mb * 2;
1792             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1793         } else {
1794             $memory = ",vgamem_mb=$vga->{memory}";
1795         }
1796     } elsif ($qxlnum && $id) {
1797         $memory = ",ram_size=67108864,vram_size=33554432";
1798     }
1799
1800     my $edidoff = "";
1801     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1802         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1803     }
1804
1805     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1806     my $vgaid = "vga" . ($id // '');
1807     my $pciaddr;
1808
1809     if ($q35 && $vgaid eq 'vga') {
1810         # the first display uses pcie.0 bus on q35 machines
1811         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1812     } else {
1813         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1814     }
1815
1816     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1817 }
1818
1819 sub parse_number_sets {
1820     my ($set) = @_;
1821     my $res = [];
1822     foreach my $part (split(/;/, $set)) {
1823         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1824             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1825             push @$res, [ $1, $2 ];
1826         } else {
1827             die "invalid range: $part\n";
1828         }
1829     }
1830     return $res;
1831 }
1832
1833 sub parse_numa {
1834     my ($data) = @_;
1835
1836     my $res = parse_property_string($numa_fmt, $data);
1837     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1838     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1839     return $res;
1840 }
1841
1842 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1843 sub parse_net {
1844     my ($data) = @_;
1845
1846     my $res = eval { parse_property_string($net_fmt, $data) };
1847     if ($@) {
1848         warn $@;
1849         return;
1850     }
1851     if (!defined($res->{macaddr})) {
1852         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1853         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1854     }
1855     return $res;
1856 }
1857
1858 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1859 sub parse_ipconfig {
1860     my ($data) = @_;
1861
1862     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1863     if ($@) {
1864         warn $@;
1865         return;
1866     }
1867
1868     if ($res->{gw} && !$res->{ip}) {
1869         warn 'gateway specified without specifying an IP address';
1870         return;
1871     }
1872     if ($res->{gw6} && !$res->{ip6}) {
1873         warn 'IPv6 gateway specified without specifying an IPv6 address';
1874         return;
1875     }
1876     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1877         warn 'gateway specified together with DHCP';
1878         return;
1879     }
1880     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1881         # gw6 + auto/dhcp
1882         warn "IPv6 gateway specified together with $res->{ip6} address";
1883         return;
1884     }
1885
1886     if (!$res->{ip} && !$res->{ip6}) {
1887         return { ip => 'dhcp', ip6 => 'dhcp' };
1888     }
1889
1890     return $res;
1891 }
1892
1893 sub print_net {
1894     my $net = shift;
1895
1896     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1897 }
1898
1899 sub add_random_macs {
1900     my ($settings) = @_;
1901
1902     foreach my $opt (keys %$settings) {
1903         next if $opt !~ m/^net(\d+)$/;
1904         my $net = parse_net($settings->{$opt});
1905         next if !$net;
1906         $settings->{$opt} = print_net($net);
1907     }
1908 }
1909
1910 sub vm_is_volid_owner {
1911     my ($storecfg, $vmid, $volid) = @_;
1912
1913     if ($volid !~  m|^/|) {
1914         my ($path, $owner);
1915         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1916         if ($owner && ($owner == $vmid)) {
1917             return 1;
1918         }
1919     }
1920
1921     return;
1922 }
1923
1924 sub vmconfig_register_unused_drive {
1925     my ($storecfg, $vmid, $conf, $drive) = @_;
1926
1927     if (drive_is_cloudinit($drive)) {
1928         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1929         warn $@ if $@;
1930     } elsif (!drive_is_cdrom($drive)) {
1931         my $volid = $drive->{file};
1932         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1933             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1934         }
1935     }
1936 }
1937
1938 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1939 my $smbios1_fmt = {
1940     uuid => {
1941         type => 'string',
1942         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1943         format_description => 'UUID',
1944         description => "Set SMBIOS1 UUID.",
1945         optional => 1,
1946     },
1947     version => {
1948         type => 'string',
1949         pattern => '[A-Za-z0-9+\/]+={0,2}',
1950         format_description => 'Base64 encoded string',
1951         description => "Set SMBIOS1 version.",
1952         optional => 1,
1953     },
1954     serial => {
1955         type => 'string',
1956         pattern => '[A-Za-z0-9+\/]+={0,2}',
1957         format_description => 'Base64 encoded string',
1958         description => "Set SMBIOS1 serial number.",
1959         optional => 1,
1960     },
1961     manufacturer => {
1962         type => 'string',
1963         pattern => '[A-Za-z0-9+\/]+={0,2}',
1964         format_description => 'Base64 encoded string',
1965         description => "Set SMBIOS1 manufacturer.",
1966         optional => 1,
1967     },
1968     product => {
1969         type => 'string',
1970         pattern => '[A-Za-z0-9+\/]+={0,2}',
1971         format_description => 'Base64 encoded string',
1972         description => "Set SMBIOS1 product ID.",
1973         optional => 1,
1974     },
1975     sku => {
1976         type => 'string',
1977         pattern => '[A-Za-z0-9+\/]+={0,2}',
1978         format_description => 'Base64 encoded string',
1979         description => "Set SMBIOS1 SKU string.",
1980         optional => 1,
1981     },
1982     family => {
1983         type => 'string',
1984         pattern => '[A-Za-z0-9+\/]+={0,2}',
1985         format_description => 'Base64 encoded string',
1986         description => "Set SMBIOS1 family string.",
1987         optional => 1,
1988     },
1989     base64 => {
1990         type => 'boolean',
1991         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
1992         optional => 1,
1993     },
1994 };
1995
1996 sub parse_smbios1 {
1997     my ($data) = @_;
1998
1999     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2000     warn $@ if $@;
2001     return $res;
2002 }
2003
2004 sub print_smbios1 {
2005     my ($smbios1) = @_;
2006     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2007 }
2008
2009 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2010
2011 sub parse_watchdog {
2012     my ($value) = @_;
2013
2014     return if !$value;
2015
2016     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2017     warn $@ if $@;
2018     return $res;
2019 }
2020
2021 sub parse_guest_agent {
2022     my ($conf) = @_;
2023
2024     return {} if !defined($conf->{agent});
2025
2026     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2027     warn $@ if $@;
2028
2029     # if the agent is disabled ignore the other potentially set properties
2030     return {} if !$res->{enabled};
2031     return $res;
2032 }
2033
2034 sub get_qga_key {
2035     my ($conf, $key) = @_;
2036     return undef if !defined($conf->{agent});
2037
2038     my $agent = parse_guest_agent($conf);
2039     return $agent->{$key};
2040 }
2041
2042 sub parse_vga {
2043     my ($value) = @_;
2044
2045     return {} if !$value;
2046     my $res = eval { parse_property_string($vga_fmt, $value) };
2047     warn $@ if $@;
2048     return $res;
2049 }
2050
2051 sub parse_rng {
2052     my ($value) = @_;
2053
2054     return if !$value;
2055
2056     my $res = eval { parse_property_string($rng_fmt, $value) };
2057     warn $@ if $@;
2058     return $res;
2059 }
2060
2061 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2062 sub verify_usb_device {
2063     my ($value, $noerr) = @_;
2064
2065     return $value if parse_usb_device($value);
2066
2067     return if $noerr;
2068
2069     die "unable to parse usb device\n";
2070 }
2071
2072 # add JSON properties for create and set function
2073 sub json_config_properties {
2074     my $prop = shift;
2075
2076     foreach my $opt (keys %$confdesc) {
2077         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' ||
2078             $opt eq 'runningmachine' || $opt eq 'runningcpu';
2079         $prop->{$opt} = $confdesc->{$opt};
2080     }
2081
2082     return $prop;
2083 }
2084
2085 # return copy of $confdesc_cloudinit to generate documentation
2086 sub cloudinit_config_properties {
2087
2088     return dclone($confdesc_cloudinit);
2089 }
2090
2091 sub check_type {
2092     my ($key, $value) = @_;
2093
2094     die "unknown setting '$key'\n" if !$confdesc->{$key};
2095
2096     my $type = $confdesc->{$key}->{type};
2097
2098     if (!defined($value)) {
2099         die "got undefined value\n";
2100     }
2101
2102     if ($value =~ m/[\n\r]/) {
2103         die "property contains a line feed\n";
2104     }
2105
2106     if ($type eq 'boolean') {
2107         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2108         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2109         die "type check ('boolean') failed - got '$value'\n";
2110     } elsif ($type eq 'integer') {
2111         return int($1) if $value =~ m/^(\d+)$/;
2112         die "type check ('integer') failed - got '$value'\n";
2113     } elsif ($type eq 'number') {
2114         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2115         die "type check ('number') failed - got '$value'\n";
2116     } elsif ($type eq 'string') {
2117         if (my $fmt = $confdesc->{$key}->{format}) {
2118             PVE::JSONSchema::check_format($fmt, $value);
2119             return $value;
2120         }
2121         $value =~ s/^\"(.*)\"$/$1/;
2122         return $value;
2123     } else {
2124         die "internal error"
2125     }
2126 }
2127
2128 sub destroy_vm {
2129     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2130
2131     my $conf = PVE::QemuConfig->load_config($vmid);
2132
2133     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2134
2135     if ($conf->{template}) {
2136         # check if any base image is still used by a linked clone
2137         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2138                 my ($ds, $drive) = @_;
2139                 return if drive_is_cdrom($drive);
2140
2141                 my $volid = $drive->{file};
2142                 return if !$volid || $volid =~ m|^/|;
2143
2144                 die "base volume '$volid' is still in use by linked cloned\n"
2145                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2146
2147         });
2148     }
2149
2150     my $remove_owned_drive = sub {
2151         my ($ds, $drive) = @_;
2152         return if drive_is_cdrom($drive, 1);
2153
2154         my $volid = $drive->{file};
2155         return if !$volid || $volid =~ m|^/|;
2156
2157         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2158         return if !$path || !$owner || ($owner != $vmid);
2159
2160         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2161         warn "Could not remove disk '$volid', check manually: $@" if $@;
2162     };
2163
2164     # only remove disks owned by this VM (referenced in the config)
2165     my $include_opts = {
2166         include_unused => 1,
2167         extra_keys => ['vmstate'],
2168     };
2169     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2170
2171     for my $snap (values %{$conf->{snapshots}}) {
2172         next if !defined($snap->{vmstate});
2173         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2174         next if !defined($drive);
2175         $remove_owned_drive->('vmstate', $drive);
2176     }
2177
2178     if ($purge_unreferenced) { # also remove unreferenced disk
2179         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2180         PVE::Storage::foreach_volid($vmdisks, sub {
2181             my ($volid, $sid, $volname, $d) = @_;
2182             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2183             warn $@ if $@;
2184         });
2185     }
2186
2187     if (defined $replacement_conf) {
2188         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2189     } else {
2190         PVE::QemuConfig->destroy_config($vmid);
2191     }
2192 }
2193
2194 sub parse_vm_config {
2195     my ($filename, $raw) = @_;
2196
2197     return if !defined($raw);
2198
2199     my $res = {
2200         digest => Digest::SHA::sha1_hex($raw),
2201         snapshots => {},
2202         pending => {},
2203     };
2204
2205     $filename =~ m|/qemu-server/(\d+)\.conf$|
2206         || die "got strange filename '$filename'";
2207
2208     my $vmid = $1;
2209
2210     my $conf = $res;
2211     my $descr;
2212     my $section = '';
2213
2214     my @lines = split(/\n/, $raw);
2215     foreach my $line (@lines) {
2216         next if $line =~ m/^\s*$/;
2217
2218         if ($line =~ m/^\[PENDING\]\s*$/i) {
2219             $section = 'pending';
2220             if (defined($descr)) {
2221                 $descr =~ s/\s+$//;
2222                 $conf->{description} = $descr;
2223             }
2224             $descr = undef;
2225             $conf = $res->{$section} = {};
2226             next;
2227
2228         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2229             $section = $1;
2230             if (defined($descr)) {
2231                 $descr =~ s/\s+$//;
2232                 $conf->{description} = $descr;
2233             }
2234             $descr = undef;
2235             $conf = $res->{snapshots}->{$section} = {};
2236             next;
2237         }
2238
2239         if ($line =~ m/^\#(.*)\s*$/) {
2240             $descr = '' if !defined($descr);
2241             $descr .= PVE::Tools::decode_text($1) . "\n";
2242             next;
2243         }
2244
2245         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2246             $descr = '' if !defined($descr);
2247             $descr .= PVE::Tools::decode_text($2);
2248         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2249             $conf->{snapstate} = $1;
2250         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2251             my $key = $1;
2252             my $value = $2;
2253             $conf->{$key} = $value;
2254         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2255             my $value = $1;
2256             if ($section eq 'pending') {
2257                 $conf->{delete} = $value; # we parse this later
2258             } else {
2259                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2260             }
2261         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2262             my $key = $1;
2263             my $value = $2;
2264             eval { $value = check_type($key, $value); };
2265             if ($@) {
2266                 warn "vm $vmid - unable to parse value of '$key' - $@";
2267             } else {
2268                 $key = 'ide2' if $key eq 'cdrom';
2269                 my $fmt = $confdesc->{$key}->{format};
2270                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2271                     my $v = parse_drive($key, $value);
2272                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2273                         $v->{file} = $volid;
2274                         $value = print_drive($v);
2275                     } else {
2276                         warn "vm $vmid - unable to parse value of '$key'\n";
2277                         next;
2278                     }
2279                 }
2280
2281                 $conf->{$key} = $value;
2282             }
2283         } else {
2284             warn "vm $vmid - unable to parse config: $line\n";
2285         }
2286     }
2287
2288     if (defined($descr)) {
2289         $descr =~ s/\s+$//;
2290         $conf->{description} = $descr;
2291     }
2292     delete $res->{snapstate}; # just to be sure
2293
2294     return $res;
2295 }
2296
2297 sub write_vm_config {
2298     my ($filename, $conf) = @_;
2299
2300     delete $conf->{snapstate}; # just to be sure
2301
2302     if ($conf->{cdrom}) {
2303         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2304         $conf->{ide2} = $conf->{cdrom};
2305         delete $conf->{cdrom};
2306     }
2307
2308     # we do not use 'smp' any longer
2309     if ($conf->{sockets}) {
2310         delete $conf->{smp};
2311     } elsif ($conf->{smp}) {
2312         $conf->{sockets} = $conf->{smp};
2313         delete $conf->{cores};
2314         delete $conf->{smp};
2315     }
2316
2317     my $used_volids = {};
2318
2319     my $cleanup_config = sub {
2320         my ($cref, $pending, $snapname) = @_;
2321
2322         foreach my $key (keys %$cref) {
2323             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2324                 $key eq 'snapstate' || $key eq 'pending';
2325             my $value = $cref->{$key};
2326             if ($key eq 'delete') {
2327                 die "propertry 'delete' is only allowed in [PENDING]\n"
2328                     if !$pending;
2329                 # fixme: check syntax?
2330                 next;
2331             }
2332             eval { $value = check_type($key, $value); };
2333             die "unable to parse value of '$key' - $@" if $@;
2334
2335             $cref->{$key} = $value;
2336
2337             if (!$snapname && is_valid_drivename($key)) {
2338                 my $drive = parse_drive($key, $value);
2339                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2340             }
2341         }
2342     };
2343
2344     &$cleanup_config($conf);
2345
2346     &$cleanup_config($conf->{pending}, 1);
2347
2348     foreach my $snapname (keys %{$conf->{snapshots}}) {
2349         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2350         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2351     }
2352
2353     # remove 'unusedX' settings if we re-add a volume
2354     foreach my $key (keys %$conf) {
2355         my $value = $conf->{$key};
2356         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2357             delete $conf->{$key};
2358         }
2359     }
2360
2361     my $generate_raw_config = sub {
2362         my ($conf, $pending) = @_;
2363
2364         my $raw = '';
2365
2366         # add description as comment to top of file
2367         if (defined(my $descr = $conf->{description})) {
2368             if ($descr) {
2369                 foreach my $cl (split(/\n/, $descr)) {
2370                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2371                 }
2372             } else {
2373                 $raw .= "#\n" if $pending;
2374             }
2375         }
2376
2377         foreach my $key (sort keys %$conf) {
2378             next if $key =~ /^(digest|description|pending|snapshots)$/;
2379             $raw .= "$key: $conf->{$key}\n";
2380         }
2381         return $raw;
2382     };
2383
2384     my $raw = &$generate_raw_config($conf);
2385
2386     if (scalar(keys %{$conf->{pending}})){
2387         $raw .= "\n[PENDING]\n";
2388         $raw .= &$generate_raw_config($conf->{pending}, 1);
2389     }
2390
2391     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2392         $raw .= "\n[$snapname]\n";
2393         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2394     }
2395
2396     return $raw;
2397 }
2398
2399 sub load_defaults {
2400
2401     my $res = {};
2402
2403     # we use static defaults from our JSON schema configuration
2404     foreach my $key (keys %$confdesc) {
2405         if (defined(my $default = $confdesc->{$key}->{default})) {
2406             $res->{$key} = $default;
2407         }
2408     }
2409
2410     return $res;
2411 }
2412
2413 sub config_list {
2414     my $vmlist = PVE::Cluster::get_vmlist();
2415     my $res = {};
2416     return $res if !$vmlist || !$vmlist->{ids};
2417     my $ids = $vmlist->{ids};
2418     my $nodename = nodename();
2419
2420     foreach my $vmid (keys %$ids) {
2421         my $d = $ids->{$vmid};
2422         next if !$d->{node} || $d->{node} ne $nodename;
2423         next if !$d->{type} || $d->{type} ne 'qemu';
2424         $res->{$vmid}->{exists} = 1;
2425     }
2426     return $res;
2427 }
2428
2429 # test if VM uses local resources (to prevent migration)
2430 sub check_local_resources {
2431     my ($conf, $noerr) = @_;
2432
2433     my @loc_res = ();
2434
2435     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2436     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2437
2438     push @loc_res, "ivshmem" if $conf->{ivshmem};
2439
2440     foreach my $k (keys %$conf) {
2441         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2442         # sockets are safe: they will recreated be on the target side post-migrate
2443         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2444         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2445     }
2446
2447     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2448
2449     return \@loc_res;
2450 }
2451
2452 # check if used storages are available on all nodes (use by migrate)
2453 sub check_storage_availability {
2454     my ($storecfg, $conf, $node) = @_;
2455
2456     PVE::QemuConfig->foreach_volume($conf, sub {
2457         my ($ds, $drive) = @_;
2458
2459         my $volid = $drive->{file};
2460         return if !$volid;
2461
2462         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2463         return if !$sid;
2464
2465         # check if storage is available on both nodes
2466         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2467         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2468
2469         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2470
2471         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2472             if !$scfg->{content}->{$vtype};
2473    });
2474 }
2475
2476 # list nodes where all VM images are available (used by has_feature API)
2477 sub shared_nodes {
2478     my ($conf, $storecfg) = @_;
2479
2480     my $nodelist = PVE::Cluster::get_nodelist();
2481     my $nodehash = { map { $_ => 1 } @$nodelist };
2482     my $nodename = nodename();
2483
2484     PVE::QemuConfig->foreach_volume($conf, sub {
2485         my ($ds, $drive) = @_;
2486
2487         my $volid = $drive->{file};
2488         return if !$volid;
2489
2490         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2491         if ($storeid) {
2492             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2493             if ($scfg->{disable}) {
2494                 $nodehash = {};
2495             } elsif (my $avail = $scfg->{nodes}) {
2496                 foreach my $node (keys %$nodehash) {
2497                     delete $nodehash->{$node} if !$avail->{$node};
2498                 }
2499             } elsif (!$scfg->{shared}) {
2500                 foreach my $node (keys %$nodehash) {
2501                     delete $nodehash->{$node} if $node ne $nodename
2502                 }
2503             }
2504         }
2505     });
2506
2507     return $nodehash
2508 }
2509
2510 sub check_local_storage_availability {
2511     my ($conf, $storecfg) = @_;
2512
2513     my $nodelist = PVE::Cluster::get_nodelist();
2514     my $nodehash = { map { $_ => {} } @$nodelist };
2515
2516     PVE::QemuConfig->foreach_volume($conf, sub {
2517         my ($ds, $drive) = @_;
2518
2519         my $volid = $drive->{file};
2520         return if !$volid;
2521
2522         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2523         if ($storeid) {
2524             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2525
2526             if ($scfg->{disable}) {
2527                 foreach my $node (keys %$nodehash) {
2528                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2529                 }
2530             } elsif (my $avail = $scfg->{nodes}) {
2531                 foreach my $node (keys %$nodehash) {
2532                     if (!$avail->{$node}) {
2533                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2534                     }
2535                 }
2536             }
2537         }
2538     });
2539
2540     foreach my $node (values %$nodehash) {
2541         if (my $unavail = $node->{unavailable_storages}) {
2542             $node->{unavailable_storages} = [ sort keys %$unavail ];
2543         }
2544     }
2545
2546     return $nodehash
2547 }
2548
2549 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2550 sub check_running {
2551     my ($vmid, $nocheck, $node) = @_;
2552
2553     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2554     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2555 }
2556
2557 sub vzlist {
2558
2559     my $vzlist = config_list();
2560
2561     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2562
2563     while (defined(my $de = $fd->read)) {
2564         next if $de !~ m/^(\d+)\.pid$/;
2565         my $vmid = $1;
2566         next if !defined($vzlist->{$vmid});
2567         if (my $pid = check_running($vmid)) {
2568             $vzlist->{$vmid}->{pid} = $pid;
2569         }
2570     }
2571
2572     return $vzlist;
2573 }
2574
2575 our $vmstatus_return_properties = {
2576     vmid => get_standard_option('pve-vmid'),
2577     status => {
2578         description => "Qemu process status.",
2579         type => 'string',
2580         enum => ['stopped', 'running'],
2581     },
2582     maxmem => {
2583         description => "Maximum memory in bytes.",
2584         type => 'integer',
2585         optional => 1,
2586         renderer => 'bytes',
2587     },
2588     maxdisk => {
2589         description => "Root disk size in bytes.",
2590         type => 'integer',
2591         optional => 1,
2592         renderer => 'bytes',
2593     },
2594     name => {
2595         description => "VM name.",
2596         type => 'string',
2597         optional => 1,
2598     },
2599     qmpstatus => {
2600         description => "Qemu QMP agent status.",
2601         type => 'string',
2602         optional => 1,
2603     },
2604     pid => {
2605         description => "PID of running qemu process.",
2606         type => 'integer',
2607         optional => 1,
2608     },
2609     uptime => {
2610         description => "Uptime.",
2611         type => 'integer',
2612         optional => 1,
2613         renderer => 'duration',
2614     },
2615     cpus => {
2616         description => "Maximum usable CPUs.",
2617         type => 'number',
2618         optional => 1,
2619     },
2620     lock => {
2621         description => "The current config lock, if any.",
2622         type => 'string',
2623         optional => 1,
2624     },
2625     tags => {
2626         description  => "The current configured tags, if any",
2627         type => 'string',
2628         optional => 1,
2629     },
2630     'running-machine' => {
2631         description => "The currently running machine type (if running).",
2632         type => 'string',
2633         optional => 1,
2634     },
2635     'running-qemu' => {
2636         description => "The currently running QEMU version (if running).",
2637         type => 'string',
2638         optional => 1,
2639     },
2640 };
2641
2642 my $last_proc_pid_stat;
2643
2644 # get VM status information
2645 # This must be fast and should not block ($full == false)
2646 # We only query KVM using QMP if $full == true (this can be slow)
2647 sub vmstatus {
2648     my ($opt_vmid, $full) = @_;
2649
2650     my $res = {};
2651
2652     my $storecfg = PVE::Storage::config();
2653
2654     my $list = vzlist();
2655     my $defaults = load_defaults();
2656
2657     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2658
2659     my $cpucount = $cpuinfo->{cpus} || 1;
2660
2661     foreach my $vmid (keys %$list) {
2662         next if $opt_vmid && ($vmid ne $opt_vmid);
2663
2664         my $conf = PVE::QemuConfig->load_config($vmid);
2665
2666         my $d = { vmid => int($vmid) };
2667         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2668
2669         # fixme: better status?
2670         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2671
2672         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2673         if (defined($size)) {
2674             $d->{disk} = 0; # no info available
2675             $d->{maxdisk} = $size;
2676         } else {
2677             $d->{disk} = 0;
2678             $d->{maxdisk} = 0;
2679         }
2680
2681         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2682             * ($conf->{cores} || $defaults->{cores});
2683         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2684         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2685
2686         $d->{name} = $conf->{name} || "VM $vmid";
2687         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2688             : $defaults->{memory}*(1024*1024);
2689
2690         if ($conf->{balloon}) {
2691             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2692             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2693                 : $defaults->{shares};
2694         }
2695
2696         $d->{uptime} = 0;
2697         $d->{cpu} = 0;
2698         $d->{mem} = 0;
2699
2700         $d->{netout} = 0;
2701         $d->{netin} = 0;
2702
2703         $d->{diskread} = 0;
2704         $d->{diskwrite} = 0;
2705
2706         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2707
2708         $d->{serial} = 1 if conf_has_serial($conf);
2709         $d->{lock} = $conf->{lock} if $conf->{lock};
2710         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2711
2712         $res->{$vmid} = $d;
2713     }
2714
2715     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2716     foreach my $dev (keys %$netdev) {
2717         next if $dev !~ m/^tap([1-9]\d*)i/;
2718         my $vmid = $1;
2719         my $d = $res->{$vmid};
2720         next if !$d;
2721
2722         $d->{netout} += $netdev->{$dev}->{receive};
2723         $d->{netin} += $netdev->{$dev}->{transmit};
2724
2725         if ($full) {
2726             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2727             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2728         }
2729
2730     }
2731
2732     my $ctime = gettimeofday;
2733
2734     foreach my $vmid (keys %$list) {
2735
2736         my $d = $res->{$vmid};
2737         my $pid = $d->{pid};
2738         next if !$pid;
2739
2740         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2741         next if !$pstat; # not running
2742
2743         my $used = $pstat->{utime} + $pstat->{stime};
2744
2745         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2746
2747         if ($pstat->{vsize}) {
2748             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2749         }
2750
2751         my $old = $last_proc_pid_stat->{$pid};
2752         if (!$old) {
2753             $last_proc_pid_stat->{$pid} = {
2754                 time => $ctime,
2755                 used => $used,
2756                 cpu => 0,
2757             };
2758             next;
2759         }
2760
2761         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2762
2763         if ($dtime > 1000) {
2764             my $dutime = $used -  $old->{used};
2765
2766             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2767             $last_proc_pid_stat->{$pid} = {
2768                 time => $ctime,
2769                 used => $used,
2770                 cpu => $d->{cpu},
2771             };
2772         } else {
2773             $d->{cpu} = $old->{cpu};
2774         }
2775     }
2776
2777     return $res if !$full;
2778
2779     my $qmpclient = PVE::QMPClient->new();
2780
2781     my $ballooncb = sub {
2782         my ($vmid, $resp) = @_;
2783
2784         my $info = $resp->{'return'};
2785         return if !$info->{max_mem};
2786
2787         my $d = $res->{$vmid};
2788
2789         # use memory assigned to VM
2790         $d->{maxmem} = $info->{max_mem};
2791         $d->{balloon} = $info->{actual};
2792
2793         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2794             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2795             $d->{freemem} = $info->{free_mem};
2796         }
2797
2798         $d->{ballooninfo} = $info;
2799     };
2800
2801     my $blockstatscb = sub {
2802         my ($vmid, $resp) = @_;
2803         my $data = $resp->{'return'} || [];
2804         my $totalrdbytes = 0;
2805         my $totalwrbytes = 0;
2806
2807         for my $blockstat (@$data) {
2808             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2809             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2810
2811             $blockstat->{device} =~ s/drive-//;
2812             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2813         }
2814         $res->{$vmid}->{diskread} = $totalrdbytes;
2815         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2816     };
2817
2818     my $machinecb = sub {
2819         my ($vmid, $resp) = @_;
2820         my $data = $resp->{'return'} || [];
2821
2822         $res->{$vmid}->{'running-machine'} =
2823             PVE::QemuServer::Machine::current_from_query_machines($data);
2824     };
2825
2826     my $versioncb = sub {
2827         my ($vmid, $resp) = @_;
2828         my $data = $resp->{'return'} // {};
2829         my $version = 'unknown';
2830
2831         if (my $v = $data->{qemu}) {
2832             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
2833         }
2834
2835         $res->{$vmid}->{'running-qemu'} = $version;
2836     };
2837
2838     my $statuscb = sub {
2839         my ($vmid, $resp) = @_;
2840
2841         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2842         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
2843         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
2844         # this fails if ballon driver is not loaded, so this must be
2845         # the last commnand (following command are aborted if this fails).
2846         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2847
2848         my $status = 'unknown';
2849         if (!defined($status = $resp->{'return'}->{status})) {
2850             warn "unable to get VM status\n";
2851             return;
2852         }
2853
2854         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2855     };
2856
2857     foreach my $vmid (keys %$list) {
2858         next if $opt_vmid && ($vmid ne $opt_vmid);
2859         next if !$res->{$vmid}->{pid}; # not running
2860         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2861     }
2862
2863     $qmpclient->queue_execute(undef, 2);
2864
2865     foreach my $vmid (keys %$list) {
2866         next if $opt_vmid && ($vmid ne $opt_vmid);
2867         next if !$res->{$vmid}->{pid}; #not running
2868
2869         # we can't use the $qmpclient since it might have already aborted on
2870         # 'query-balloon', but this might also fail for older versions...
2871         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
2872         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
2873     }
2874
2875     foreach my $vmid (keys %$list) {
2876         next if $opt_vmid && ($vmid ne $opt_vmid);
2877         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2878     }
2879
2880     return $res;
2881 }
2882
2883 sub conf_has_serial {
2884     my ($conf) = @_;
2885
2886     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2887         if ($conf->{"serial$i"}) {
2888             return 1;
2889         }
2890     }
2891
2892     return 0;
2893 }
2894
2895 sub conf_has_audio {
2896     my ($conf, $id) = @_;
2897
2898     $id //= 0;
2899     my $audio = $conf->{"audio$id"};
2900     return if !defined($audio);
2901
2902     my $audioproperties = parse_property_string($audio_fmt, $audio);
2903     my $audiodriver = $audioproperties->{driver} // 'spice';
2904
2905     return {
2906         dev => $audioproperties->{device},
2907         dev_id => "audiodev$id",
2908         backend => $audiodriver,
2909         backend_id => "$audiodriver-backend${id}",
2910     };
2911 }
2912
2913 sub audio_devs {
2914     my ($audio, $audiopciaddr, $machine_version) = @_;
2915
2916     my $devs = [];
2917
2918     my $id = $audio->{dev_id};
2919     my $audiodev = "";
2920     if (min_version($machine_version, 4, 2)) {
2921         $audiodev = ",audiodev=$audio->{backend_id}";
2922     }
2923
2924     if ($audio->{dev} eq 'AC97') {
2925         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
2926     } elsif ($audio->{dev} =~ /intel\-hda$/) {
2927         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
2928         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
2929         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
2930     } else {
2931         die "unkown audio device '$audio->{dev}', implement me!";
2932     }
2933
2934     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
2935
2936     return $devs;
2937 }
2938
2939 sub vga_conf_has_spice {
2940     my ($vga) = @_;
2941
2942     my $vgaconf = parse_vga($vga);
2943     my $vgatype = $vgaconf->{type};
2944     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
2945
2946     return $1 || 1;
2947 }
2948
2949 sub is_native($) {
2950     my ($arch) = @_;
2951     return get_host_arch() eq $arch;
2952 }
2953
2954 sub get_vm_arch {
2955     my ($conf) = @_;
2956     return $conf->{arch} // get_host_arch();
2957 }
2958
2959 my $default_machines = {
2960     x86_64 => 'pc',
2961     aarch64 => 'virt',
2962 };
2963
2964 sub get_installed_machine_version {
2965     my ($kvmversion) = @_;
2966     $kvmversion = kvm_user_version() if !defined($kvmversion);
2967     $kvmversion =~ m/^(\d+\.\d+)/;
2968     return $1;
2969 }
2970
2971 sub windows_get_pinned_machine_version {
2972     my ($machine, $base_version, $kvmversion) = @_;
2973
2974     my $pin_version = $base_version;
2975     if (!defined($base_version) ||
2976         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
2977     ) {
2978         $pin_version = get_installed_machine_version($kvmversion);
2979     }
2980     if (!$machine || $machine eq 'pc') {
2981         $machine = "pc-i440fx-$pin_version";
2982     } elsif ($machine eq 'q35') {
2983         $machine = "pc-q35-$pin_version";
2984     } elsif ($machine eq 'virt') {
2985         $machine = "virt-$pin_version";
2986     } else {
2987         warn "unknown machine type '$machine', not touching that!\n";
2988     }
2989
2990     return $machine;
2991 }
2992
2993 sub get_vm_machine {
2994     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
2995
2996     my $machine = $forcemachine || $conf->{machine};
2997
2998     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
2999         $kvmversion //= kvm_user_version();
3000         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3001         # layout which confuses windows quite a bit and may result in various regressions..
3002         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3003         if (windows_version($conf->{ostype})) {
3004             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3005         }
3006         $arch //= 'x86_64';
3007         $machine ||= $default_machines->{$arch};
3008         if ($add_pve_version) {
3009             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3010             $machine .= "+pve$pvever";
3011         }
3012     }
3013
3014     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3015         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3016         $machine = $1 if $is_pxe;
3017
3018         # for version-pinned machines that do not include a pve-version (e.g.
3019         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3020         $machine .= '+pve0';
3021
3022         $machine .= '.pxe' if $is_pxe;
3023     }
3024
3025     return $machine;
3026 }
3027
3028 sub get_ovmf_files($) {
3029     my ($arch) = @_;
3030
3031     my $ovmf = $OVMF->{$arch}
3032         or die "no OVMF images known for architecture '$arch'\n";
3033
3034     return @$ovmf;
3035 }
3036
3037 my $Arch2Qemu = {
3038     aarch64 => '/usr/bin/qemu-system-aarch64',
3039     x86_64 => '/usr/bin/qemu-system-x86_64',
3040 };
3041 sub get_command_for_arch($) {
3042     my ($arch) = @_;
3043     return '/usr/bin/kvm' if is_native($arch);
3044
3045     my $cmd = $Arch2Qemu->{$arch}
3046         or die "don't know how to emulate architecture '$arch'\n";
3047     return $cmd;
3048 }
3049
3050 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3051 # to use in a QEMU command line (-cpu element), first array_intersect the result
3052 # of query_supported_ with query_understood_. This is necessary because:
3053 #
3054 # a) query_understood_ returns flags the host cannot use and
3055 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3056 #    flags, but CPU settings - with most of them being flags. Those settings
3057 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3058 #
3059 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3060 # expensive. If you need the value returned from this, you can get it much
3061 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3062 # $accel being 'kvm' or 'tcg'.
3063 #
3064 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3065 # changes, automatically populating pmxcfs.
3066 #
3067 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3068 # since kvm and tcg machines support different flags
3069 #
3070 sub query_supported_cpu_flags {
3071     my ($arch) = @_;
3072
3073     $arch //= get_host_arch();
3074     my $default_machine = $default_machines->{$arch};
3075
3076     my $flags = {};
3077
3078     # FIXME: Once this is merged, the code below should work for ARM as well:
3079     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3080     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3081         $arch eq "aarch64";
3082
3083     my $kvm_supported = defined(kvm_version());
3084     my $qemu_cmd = get_command_for_arch($arch);
3085     my $fakevmid = -1;
3086     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3087
3088     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3089     my $query_supported_run_qemu = sub {
3090         my ($kvm) = @_;
3091
3092         my $flags = {};
3093         my $cmd = [
3094             $qemu_cmd,
3095             '-machine', $default_machine,
3096             '-display', 'none',
3097             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3098             '-mon', 'chardev=qmp,mode=control',
3099             '-pidfile', $pidfile,
3100             '-S', '-daemonize'
3101         ];
3102
3103         if (!$kvm) {
3104             push @$cmd, '-accel', 'tcg';
3105         }
3106
3107         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3108         die "QEMU flag querying VM exited with code " . $rc if $rc;
3109
3110         eval {
3111             my $cmd_result = mon_cmd(
3112                 $fakevmid,
3113                 'query-cpu-model-expansion',
3114                 type => 'full',
3115                 model => { name => 'host' }
3116             );
3117
3118             my $props = $cmd_result->{model}->{props};
3119             foreach my $prop (keys %$props) {
3120                 next if $props->{$prop} ne '1';
3121                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3122                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3123                 # We only keep those with underscores, to match /proc/cpuinfo
3124                 $prop =~ s/\.|-/_/g;
3125                 $flags->{$prop} = 1;
3126             }
3127         };
3128         my $err = $@;
3129
3130         # force stop with 10 sec timeout and 'nocheck'
3131         # always stop, even if QMP failed
3132         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3133
3134         die $err if $err;
3135
3136         return [ sort keys %$flags ];
3137     };
3138
3139     # We need to query QEMU twice, since KVM and TCG have different supported flags
3140     PVE::QemuConfig->lock_config($fakevmid, sub {
3141         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3142         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3143
3144         if ($kvm_supported) {
3145             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3146             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3147         }
3148     });
3149
3150     return $flags;
3151 }
3152
3153 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3154 my $understood_cpu_flag_dir = "/usr/share/kvm";
3155 sub query_understood_cpu_flags {
3156     my $arch = get_host_arch();
3157     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3158
3159     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3160         if ! -e $filepath;
3161
3162     my $raw = file_get_contents($filepath);
3163     $raw =~ s/^\s+|\s+$//g;
3164     my @flags = split(/\s+/, $raw);
3165
3166     return \@flags;
3167 }
3168
3169 sub config_to_command {
3170     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3171         $pbs_backing) = @_;
3172
3173     my $cmd = [];
3174     my $globalFlags = [];
3175     my $machineFlags = [];
3176     my $rtcFlags = [];
3177     my $devices = [];
3178     my $pciaddr = '';
3179     my $bridges = {};
3180     my $ostype = $conf->{ostype};
3181     my $winversion = windows_version($ostype);
3182     my $kvm = $conf->{kvm};
3183     my $nodename = nodename();
3184
3185     my $arch = get_vm_arch($conf);
3186     my $kvm_binary = get_command_for_arch($arch);
3187     my $kvmver = kvm_user_version($kvm_binary);
3188
3189     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3190         $kvmver //= "undefined";
3191         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3192     }
3193
3194     my $add_pve_version = min_version($kvmver, 4, 1);
3195
3196     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3197     my $machine_version = extract_version($machine_type, $kvmver);
3198     $kvm //= 1 if is_native($arch);
3199
3200     $machine_version =~ m/(\d+)\.(\d+)/;
3201     my ($machine_major, $machine_minor) = ($1, $2);
3202
3203     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3204         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3205     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3206         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3207             ." please upgrade node '$nodename'\n"
3208     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3209         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3210         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3211             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3212             ." node '$nodename'\n";
3213     }
3214
3215     # if a specific +pve version is required for a feature, use $version_guard
3216     # instead of min_version to allow machines to be run with the minimum
3217     # required version
3218     my $required_pve_version = 0;
3219     my $version_guard = sub {
3220         my ($major, $minor, $pve) = @_;
3221         return 0 if !min_version($machine_version, $major, $minor, $pve);
3222         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3223         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3224         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3225         return 1;
3226     };
3227
3228     if ($kvm && !defined kvm_version()) {
3229         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3230             ." or enable in BIOS.\n";
3231     }
3232
3233     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3234     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3235     my $use_old_bios_files = undef;
3236     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3237
3238     my $cpuunits = defined($conf->{cpuunits}) ?
3239             $conf->{cpuunits} : $defaults->{cpuunits};
3240
3241     push @$cmd, $kvm_binary;
3242
3243     push @$cmd, '-id', $vmid;
3244
3245     my $vmname = $conf->{name} || "vm$vmid";
3246
3247     push @$cmd, '-name', $vmname;
3248
3249     push @$cmd, '-no-shutdown';
3250
3251     my $use_virtio = 0;
3252
3253     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3254     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3255     push @$cmd, '-mon', "chardev=qmp,mode=control";
3256
3257     if (min_version($machine_version, 2, 12)) {
3258         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3259         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3260     }
3261
3262     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3263
3264     push @$cmd, '-daemonize';
3265
3266     if ($conf->{smbios1}) {
3267         my $smbios_conf = parse_smbios1($conf->{smbios1});
3268         if ($smbios_conf->{base64}) {
3269             # Do not pass base64 flag to qemu
3270             delete $smbios_conf->{base64};
3271             my $smbios_string = "";
3272             foreach my $key (keys %$smbios_conf) {
3273                 my $value;
3274                 if ($key eq "uuid") {
3275                     $value = $smbios_conf->{uuid}
3276                 } else {
3277                     $value = decode_base64($smbios_conf->{$key});
3278                 }
3279                 # qemu accepts any binary data, only commas need escaping by double comma
3280                 $value =~ s/,/,,/g;
3281                 $smbios_string .= "," . $key . "=" . $value if $value;
3282             }
3283             push @$cmd, '-smbios', "type=1" . $smbios_string;
3284         } else {
3285             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3286         }
3287     }
3288
3289     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3290         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3291         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3292
3293         my ($path, $format);
3294         if (my $efidisk = $conf->{efidisk0}) {
3295             my $d = parse_drive('efidisk0', $efidisk);
3296             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3297             $format = $d->{format};
3298             if ($storeid) {
3299                 $path = PVE::Storage::path($storecfg, $d->{file});
3300                 if (!defined($format)) {
3301                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3302                     $format = qemu_img_format($scfg, $volname);
3303                 }
3304             } else {
3305                 $path = $d->{file};
3306                 die "efidisk format must be specified\n"
3307                     if !defined($format);
3308             }
3309         } else {
3310             warn "no efidisk configured! Using temporary efivars disk.\n";
3311             $path = "/tmp/$vmid-ovmf.fd";
3312             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3313             $format = 'raw';
3314         }
3315
3316         my $size_str = "";
3317
3318         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3319             $size_str = ",size=" . (-s $ovmf_vars);
3320         }
3321
3322         # on slower ceph clusters, booting without cache on efidisk can take a while, see #3329
3323         my $cache = $path =~ m/^rbd:/ ? ',cache=writeback' : '';
3324
3325         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code";
3326         push @$cmd, '-drive', "if=pflash,unit=1$cache,format=$format,id=drive-efidisk0$size_str,file=$path";
3327     }
3328
3329     # load q35 config
3330     if ($q35) {
3331         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3332         if (min_version($machine_version, 4, 0)) {
3333             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3334         } else {
3335             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3336         }
3337     }
3338
3339     if ($conf->{vmgenid}) {
3340         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3341     }
3342
3343     # add usb controllers
3344     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3345         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3346     push @$devices, @usbcontrollers if @usbcontrollers;
3347     my $vga = parse_vga($conf->{vga});
3348
3349     my $qxlnum = vga_conf_has_spice($conf->{vga});
3350     $vga->{type} = 'qxl' if $qxlnum;
3351
3352     if (!$vga->{type}) {
3353         if ($arch eq 'aarch64') {
3354             $vga->{type} = 'virtio';
3355         } elsif (min_version($machine_version, 2, 9)) {
3356             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3357         } else {
3358             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3359         }
3360     }
3361
3362     # enable absolute mouse coordinates (needed by vnc)
3363     my $tablet;
3364     if (defined($conf->{tablet})) {
3365         $tablet = $conf->{tablet};
3366     } else {
3367         $tablet = $defaults->{tablet};
3368         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3369         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3370     }
3371
3372     if ($tablet) {
3373         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3374         my $kbd = print_keyboarddevice_full($conf, $arch);
3375         push @$devices, '-device', $kbd if defined($kbd);
3376     }
3377
3378     my $bootorder = device_bootorder($conf);
3379
3380     # host pci device passthrough
3381     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3382         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3383
3384     # usb devices
3385     my $usb_dev_features = {};
3386     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3387
3388     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3389         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3390     push @$devices, @usbdevices if @usbdevices;
3391
3392     # serial devices
3393     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3394         if (my $path = $conf->{"serial$i"}) {
3395             if ($path eq 'socket') {
3396                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3397                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3398                 # On aarch64, serial0 is the UART device. Qemu only allows
3399                 # connecting UART devices via the '-serial' command line, as
3400                 # the device has a fixed slot on the hardware...
3401                 if ($arch eq 'aarch64' && $i == 0) {
3402                     push @$devices, '-serial', "chardev:serial$i";
3403                 } else {
3404                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3405                 }
3406             } else {
3407                 die "no such serial device\n" if ! -c $path;
3408                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3409                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3410             }
3411         }
3412     }
3413
3414     # parallel devices
3415     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3416         if (my $path = $conf->{"parallel$i"}) {
3417             die "no such parallel device\n" if ! -c $path;
3418             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3419             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3420             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3421         }
3422     }
3423
3424     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3425         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3426         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3427         push @$devices, @$audio_devs;
3428     }
3429
3430     my $sockets = 1;
3431     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3432     $sockets = $conf->{sockets} if  $conf->{sockets};
3433
3434     my $cores = $conf->{cores} || 1;
3435
3436     my $maxcpus = $sockets * $cores;
3437
3438     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3439
3440     my $allowed_vcpus = $cpuinfo->{cpus};
3441
3442     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3443         if ($allowed_vcpus < $maxcpus);
3444
3445     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3446
3447         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3448         for (my $i = 2; $i <= $vcpus; $i++)  {
3449             my $cpustr = print_cpu_device($conf,$i);
3450             push @$cmd, '-device', $cpustr;
3451         }
3452
3453     } else {
3454
3455         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3456     }
3457     push @$cmd, '-nodefaults';
3458
3459     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3460
3461     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3462
3463     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3464
3465     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3466         push @$devices, '-device', print_vga_device(
3467             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3468         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3469         push @$cmd,  '-vnc', "unix:$socket,password=on";
3470     } else {
3471         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3472         push @$cmd, '-nographic';
3473     }
3474
3475     # time drift fix
3476     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3477     my $useLocaltime = $conf->{localtime};
3478
3479     if ($winversion >= 5) { # windows
3480         $useLocaltime = 1 if !defined($conf->{localtime});
3481
3482         # use time drift fix when acpi is enabled
3483         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3484             $tdf = 1 if !defined($conf->{tdf});
3485         }
3486     }
3487
3488     if ($winversion >= 6) {
3489         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3490         push @$cmd, '-no-hpet';
3491     }
3492
3493     push @$rtcFlags, 'driftfix=slew' if $tdf;
3494
3495     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3496         push @$rtcFlags, "base=$conf->{startdate}";
3497     } elsif ($useLocaltime) {
3498         push @$rtcFlags, 'base=localtime';
3499     }
3500
3501     if ($forcecpu) {
3502         push @$cmd, '-cpu', $forcecpu;
3503     } else {
3504         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3505     }
3506
3507     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3508
3509     push @$cmd, '-S' if $conf->{freeze};
3510
3511     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3512
3513     my $guest_agent = parse_guest_agent($conf);
3514
3515     if ($guest_agent->{enabled}) {
3516         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3517         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3518
3519         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3520             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3521             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3522             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3523         } elsif ($guest_agent->{type} eq 'isa') {
3524             push @$devices, '-device', "isa-serial,chardev=qga0";
3525         }
3526     }
3527
3528     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3529     if ($rng && $version_guard->(4, 1, 2)) {
3530         check_rng_source($rng->{source});
3531
3532         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3533         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3534         my $limiter_str = "";
3535         if ($max_bytes) {
3536             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3537         }
3538
3539         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3540         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3541         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3542     }
3543
3544     my $spice_port;
3545
3546     if ($qxlnum) {
3547         if ($qxlnum > 1) {
3548             if ($winversion){
3549                 for (my $i = 1; $i < $qxlnum; $i++){
3550                     push @$devices, '-device', print_vga_device(
3551                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3552                 }
3553             } else {
3554                 # assume other OS works like Linux
3555                 my ($ram, $vram) = ("134217728", "67108864");
3556                 if ($vga->{memory}) {
3557                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3558                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3559                 }
3560                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3561                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3562             }
3563         }
3564
3565         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3566
3567         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3568         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3569         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3570
3571         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3572         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3573         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3574
3575         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3576         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3577
3578         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3579         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3580         if ($spice_enhancement->{foldersharing}) {
3581             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3582             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3583         }
3584
3585         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3586         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3587             if $spice_enhancement->{videostreaming};
3588
3589         push @$devices, '-spice', "$spice_opts";
3590     }
3591
3592     # enable balloon by default, unless explicitly disabled
3593     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3594         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3595         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3596     }
3597
3598     if ($conf->{watchdog}) {
3599         my $wdopts = parse_watchdog($conf->{watchdog});
3600         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3601         my $watchdog = $wdopts->{model} || 'i6300esb';
3602         push @$devices, '-device', "$watchdog$pciaddr";
3603         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3604     }
3605
3606     my $vollist = [];
3607     my $scsicontroller = {};
3608     my $ahcicontroller = {};
3609     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3610
3611     # Add iscsi initiator name if available
3612     if (my $initiator = get_initiator_name()) {
3613         push @$devices, '-iscsi', "initiator-name=$initiator";
3614     }
3615
3616     PVE::QemuConfig->foreach_volume($conf, sub {
3617         my ($ds, $drive) = @_;
3618
3619         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3620             check_volume_storage_type($storecfg, $drive->{file});
3621             push @$vollist, $drive->{file};
3622         }
3623
3624         # ignore efidisk here, already added in bios/fw handling code above
3625         return if $drive->{interface} eq 'efidisk';
3626
3627         $use_virtio = 1 if $ds =~ m/^virtio/;
3628
3629         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3630
3631         if ($drive->{interface} eq 'virtio'){
3632            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3633         }
3634
3635         if ($drive->{interface} eq 'scsi') {
3636
3637             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3638
3639             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3640                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3641
3642             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3643             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3644
3645             my $iothread = '';
3646             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3647                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3648                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3649             } elsif ($drive->{iothread}) {
3650                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3651             }
3652
3653             my $queues = '';
3654             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3655                 $queues = ",num_queues=$drive->{queues}";
3656             }
3657
3658             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3659                 if !$scsicontroller->{$controller};
3660             $scsicontroller->{$controller}=1;
3661         }
3662
3663         if ($drive->{interface} eq 'sata') {
3664             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3665             $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3666             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3667                 if !$ahcicontroller->{$controller};
3668             $ahcicontroller->{$controller}=1;
3669         }
3670
3671         my $pbs_conf = $pbs_backing->{$ds};
3672         my $pbs_name = undef;
3673         if ($pbs_conf) {
3674             $pbs_name = "drive-$ds-pbs";
3675             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
3676         }
3677
3678         my $drive_cmd = print_drive_commandline_full(
3679             $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
3680
3681         # extra protection for templates, but SATA and IDE don't support it..
3682         my $read_only = PVE::QemuConfig->is_template($conf)
3683             && $drive->{interface} ne 'sata'
3684             && $drive->{interface} ne 'ide';
3685
3686         $drive_cmd .= ',readonly=on' if $read_only;
3687
3688         push @$devices, '-drive',$drive_cmd;
3689         push @$devices, '-device', print_drivedevice_full(
3690             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3691     });
3692
3693     for (my $i = 0; $i < $MAX_NETS; $i++) {
3694         my $netname = "net$i";
3695
3696         next if !$conf->{$netname};
3697         my $d = parse_net($conf->{$netname});
3698         next if !$d;
3699
3700         $use_virtio = 1 if $d->{model} eq 'virtio';
3701
3702         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
3703
3704         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
3705         push @$devices, '-netdev', $netdevfull;
3706
3707         my $netdevicefull = print_netdevice_full(
3708             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
3709
3710         push @$devices, '-device', $netdevicefull;
3711     }
3712
3713     if ($conf->{ivshmem}) {
3714         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3715
3716         my $bus;
3717         if ($q35) {
3718             $bus = print_pcie_addr("ivshmem");
3719         } else {
3720             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3721         }
3722
3723         my $ivshmem_name = $ivshmem->{name} // $vmid;
3724         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3725
3726         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3727         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
3728             .",size=$ivshmem->{size}M";
3729     }
3730
3731     # pci.4 is nested in pci.1
3732     $bridges->{1} = 1 if $bridges->{4};
3733
3734     if (!$q35) {
3735         # add pci bridges
3736         if (min_version($machine_version, 2, 3)) {
3737            $bridges->{1} = 1;
3738            $bridges->{2} = 1;
3739         }
3740
3741         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3742
3743     }
3744
3745     for my $k (sort {$b cmp $a} keys %$bridges) {
3746         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3747
3748         my $k_name = $k;
3749         if ($k == 2 && $legacy_igd) {
3750             $k_name = "$k-igd";
3751         }
3752         $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
3753
3754         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3755         if ($q35) {
3756             # add after -readconfig pve-q35.cfg
3757             splice @$devices, 2, 0, '-device', $devstr;
3758         } else {
3759             unshift @$devices, '-device', $devstr if $k > 0;
3760         }
3761     }
3762
3763     if (!$kvm) {
3764         push @$machineFlags, 'accel=tcg';
3765     }
3766
3767     my $machine_type_min = $machine_type;
3768     if ($add_pve_version) {
3769         $machine_type_min =~ s/\+pve\d+$//;
3770         $machine_type_min .= "+pve$required_pve_version";
3771     }
3772     push @$machineFlags, "type=${machine_type_min}";
3773
3774     push @$cmd, @$devices;
3775     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
3776     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
3777     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
3778
3779     if (my $vmstate = $conf->{vmstate}) {
3780         my $statepath = PVE::Storage::path($storecfg, $vmstate);
3781         push @$vollist, $vmstate;
3782         push @$cmd, '-loadstate', $statepath;
3783         print "activating and using '$vmstate' as vmstate\n";
3784     }
3785
3786     # add custom args
3787     if ($conf->{args}) {
3788         my $aa = PVE::Tools::split_args($conf->{args});
3789         push @$cmd, @$aa;
3790     }
3791
3792     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3793 }
3794
3795 sub check_rng_source {
3796     my ($source) = @_;
3797
3798     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3799     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
3800         if ! -e $source;
3801
3802     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
3803     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
3804         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
3805         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
3806         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
3807             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
3808             ." to the host.\n";
3809     }
3810 }
3811
3812 sub spice_port {
3813     my ($vmid) = @_;
3814
3815     my $res = mon_cmd($vmid, 'query-spice');
3816
3817     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3818 }
3819
3820 sub vm_devices_list {
3821     my ($vmid) = @_;
3822
3823     my $res = mon_cmd($vmid, 'query-pci');
3824     my $devices_to_check = [];
3825     my $devices = {};
3826     foreach my $pcibus (@$res) {
3827         push @$devices_to_check, @{$pcibus->{devices}},
3828     }
3829
3830     while (@$devices_to_check) {
3831         my $to_check = [];
3832         for my $d (@$devices_to_check) {
3833             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3834             next if !$d->{'pci_bridge'};
3835
3836             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3837             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3838         }
3839         $devices_to_check = $to_check;
3840     }
3841
3842     my $resblock = mon_cmd($vmid, 'query-block');
3843     foreach my $block (@$resblock) {
3844         if($block->{device} =~ m/^drive-(\S+)/){
3845                 $devices->{$1} = 1;
3846         }
3847     }
3848
3849     my $resmice = mon_cmd($vmid, 'query-mice');
3850     foreach my $mice (@$resmice) {
3851         if ($mice->{name} eq 'QEMU HID Tablet') {
3852             $devices->{tablet} = 1;
3853             last;
3854         }
3855     }
3856
3857     # for usb devices there is no query-usb
3858     # but we can iterate over the entries in
3859     # qom-list path=/machine/peripheral
3860     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3861     foreach my $per (@$resperipheral) {
3862         if ($per->{name} =~ m/^usb\d+$/) {
3863             $devices->{$per->{name}} = 1;
3864         }
3865     }
3866
3867     return $devices;
3868 }
3869
3870 sub vm_deviceplug {
3871     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3872
3873     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3874
3875     my $devices_list = vm_devices_list($vmid);
3876     return 1 if defined($devices_list->{$deviceid});
3877
3878     # add PCI bridge if we need it for the device
3879     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
3880
3881     if ($deviceid eq 'tablet') {
3882
3883         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
3884
3885     } elsif ($deviceid eq 'keyboard') {
3886
3887         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
3888
3889     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3890
3891         die "usb hotplug currently not reliable\n";
3892         # since we can't reliably hot unplug all added usb devices and usb
3893         # passthrough breaks live migration we disable usb hotplugging for now
3894         #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3895
3896     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3897
3898         qemu_iothread_add($vmid, $deviceid, $device);
3899
3900         qemu_driveadd($storecfg, $vmid, $device);
3901         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3902
3903         qemu_deviceadd($vmid, $devicefull);
3904         eval { qemu_deviceaddverify($vmid, $deviceid); };
3905         if (my $err = $@) {
3906             eval { qemu_drivedel($vmid, $deviceid); };
3907             warn $@ if $@;
3908             die $err;
3909         }
3910
3911     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3912
3913
3914         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3915         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3916         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3917
3918         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3919
3920         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3921             qemu_iothread_add($vmid, $deviceid, $device);
3922             $devicefull .= ",iothread=iothread-$deviceid";
3923         }
3924
3925         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3926             $devicefull .= ",num_queues=$device->{queues}";
3927         }
3928
3929         qemu_deviceadd($vmid, $devicefull);
3930         qemu_deviceaddverify($vmid, $deviceid);
3931
3932     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3933
3934         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
3935         qemu_driveadd($storecfg, $vmid, $device);
3936
3937         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3938         eval { qemu_deviceadd($vmid, $devicefull); };
3939         if (my $err = $@) {
3940             eval { qemu_drivedel($vmid, $deviceid); };
3941             warn $@ if $@;
3942             die $err;
3943         }
3944
3945     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3946
3947         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
3948
3949         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
3950         my $use_old_bios_files = undef;
3951         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3952
3953         my $netdevicefull = print_netdevice_full(
3954             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
3955         qemu_deviceadd($vmid, $netdevicefull);
3956         eval {
3957             qemu_deviceaddverify($vmid, $deviceid);
3958             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
3959         };
3960         if (my $err = $@) {
3961             eval { qemu_netdevdel($vmid, $deviceid); };
3962             warn $@ if $@;
3963             die $err;
3964         }
3965
3966     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3967
3968         my $bridgeid = $2;
3969         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3970         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3971
3972         qemu_deviceadd($vmid, $devicefull);
3973         qemu_deviceaddverify($vmid, $deviceid);
3974
3975     } else {
3976         die "can't hotplug device '$deviceid'\n";
3977     }
3978
3979     return 1;
3980 }
3981
3982 # fixme: this should raise exceptions on error!
3983 sub vm_deviceunplug {
3984     my ($vmid, $conf, $deviceid) = @_;
3985
3986     my $devices_list = vm_devices_list($vmid);
3987     return 1 if !defined($devices_list->{$deviceid});
3988
3989     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
3990     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
3991
3992     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
3993
3994         qemu_devicedel($vmid, $deviceid);
3995
3996     } elsif ($deviceid =~ m/^usb\d+$/) {
3997
3998         die "usb hotplug currently not reliable\n";
3999         # when unplugging usb devices this way, there may be remaining usb
4000         # controllers/hubs so we disable it for now
4001         #qemu_devicedel($vmid, $deviceid);
4002         #qemu_devicedelverify($vmid, $deviceid);
4003
4004     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4005
4006         qemu_devicedel($vmid, $deviceid);
4007         qemu_devicedelverify($vmid, $deviceid);
4008         qemu_drivedel($vmid, $deviceid);
4009         qemu_iothread_del($conf, $vmid, $deviceid);
4010
4011     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4012
4013         qemu_devicedel($vmid, $deviceid);
4014         qemu_devicedelverify($vmid, $deviceid);
4015         qemu_iothread_del($conf, $vmid, $deviceid);
4016
4017     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4018
4019         qemu_devicedel($vmid, $deviceid);
4020         qemu_drivedel($vmid, $deviceid);
4021         qemu_deletescsihw($conf, $vmid, $deviceid);
4022
4023     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4024
4025         qemu_devicedel($vmid, $deviceid);
4026         qemu_devicedelverify($vmid, $deviceid);
4027         qemu_netdevdel($vmid, $deviceid);
4028
4029     } else {
4030         die "can't unplug device '$deviceid'\n";
4031     }
4032
4033     return 1;
4034 }
4035
4036 sub qemu_deviceadd {
4037     my ($vmid, $devicefull) = @_;
4038
4039     $devicefull = "driver=".$devicefull;
4040     my %options =  split(/[=,]/, $devicefull);
4041
4042     mon_cmd($vmid, "device_add" , %options);
4043 }
4044
4045 sub qemu_devicedel {
4046     my ($vmid, $deviceid) = @_;
4047
4048     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4049 }
4050
4051 sub qemu_iothread_add {
4052     my($vmid, $deviceid, $device) = @_;
4053
4054     if ($device->{iothread}) {
4055         my $iothreads = vm_iothreads_list($vmid);
4056         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4057     }
4058 }
4059
4060 sub qemu_iothread_del {
4061     my($conf, $vmid, $deviceid) = @_;
4062
4063     my $confid = $deviceid;
4064     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4065         $confid = 'scsi' . $1;
4066     }
4067     my $device = parse_drive($confid, $conf->{$confid});
4068     if ($device->{iothread}) {
4069         my $iothreads = vm_iothreads_list($vmid);
4070         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4071     }
4072 }
4073
4074 sub qemu_objectadd {
4075     my($vmid, $objectid, $qomtype) = @_;
4076
4077     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4078
4079     return 1;
4080 }
4081
4082 sub qemu_objectdel {
4083     my($vmid, $objectid) = @_;
4084
4085     mon_cmd($vmid, "object-del", id => $objectid);
4086
4087     return 1;
4088 }
4089
4090 sub qemu_driveadd {
4091     my ($storecfg, $vmid, $device) = @_;
4092
4093     my $kvmver = get_running_qemu_version($vmid);
4094     my $io_uring = min_version($kvmver, 6, 0);
4095     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4096     $drive =~ s/\\/\\\\/g;
4097     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4098
4099     # If the command succeeds qemu prints: "OK"
4100     return 1 if $ret =~ m/OK/s;
4101
4102     die "adding drive failed: $ret\n";
4103 }
4104
4105 sub qemu_drivedel {
4106     my($vmid, $deviceid) = @_;
4107
4108     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4109     $ret =~ s/^\s+//;
4110
4111     return 1 if $ret eq "";
4112
4113     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4114     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4115
4116     die "deleting drive $deviceid failed : $ret\n";
4117 }
4118
4119 sub qemu_deviceaddverify {
4120     my ($vmid, $deviceid) = @_;
4121
4122     for (my $i = 0; $i <= 5; $i++) {
4123          my $devices_list = vm_devices_list($vmid);
4124          return 1 if defined($devices_list->{$deviceid});
4125          sleep 1;
4126     }
4127
4128     die "error on hotplug device '$deviceid'\n";
4129 }
4130
4131
4132 sub qemu_devicedelverify {
4133     my ($vmid, $deviceid) = @_;
4134
4135     # need to verify that the device is correctly removed as device_del
4136     # is async and empty return is not reliable
4137
4138     for (my $i = 0; $i <= 5; $i++) {
4139          my $devices_list = vm_devices_list($vmid);
4140          return 1 if !defined($devices_list->{$deviceid});
4141          sleep 1;
4142     }
4143
4144     die "error on hot-unplugging device '$deviceid'\n";
4145 }
4146
4147 sub qemu_findorcreatescsihw {
4148     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4149
4150     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4151
4152     my $scsihwid="$controller_prefix$controller";
4153     my $devices_list = vm_devices_list($vmid);
4154
4155     if(!defined($devices_list->{$scsihwid})) {
4156         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4157     }
4158
4159     return 1;
4160 }
4161
4162 sub qemu_deletescsihw {
4163     my ($conf, $vmid, $opt) = @_;
4164
4165     my $device = parse_drive($opt, $conf->{$opt});
4166
4167     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4168         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4169         return 1;
4170     }
4171
4172     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4173
4174     my $devices_list = vm_devices_list($vmid);
4175     foreach my $opt (keys %{$devices_list}) {
4176         if (is_valid_drivename($opt)) {
4177             my $drive = parse_drive($opt, $conf->{$opt});
4178             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4179                 return 1;
4180             }
4181         }
4182     }
4183
4184     my $scsihwid="scsihw$controller";
4185
4186     vm_deviceunplug($vmid, $conf, $scsihwid);
4187
4188     return 1;
4189 }
4190
4191 sub qemu_add_pci_bridge {
4192     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4193
4194     my $bridges = {};
4195
4196     my $bridgeid;
4197
4198     print_pci_addr($device, $bridges, $arch, $machine_type);
4199
4200     while (my ($k, $v) = each %$bridges) {
4201         $bridgeid = $k;
4202     }
4203     return 1 if !defined($bridgeid) || $bridgeid < 1;
4204
4205     my $bridge = "pci.$bridgeid";
4206     my $devices_list = vm_devices_list($vmid);
4207
4208     if (!defined($devices_list->{$bridge})) {
4209         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4210     }
4211
4212     return 1;
4213 }
4214
4215 sub qemu_set_link_status {
4216     my ($vmid, $device, $up) = @_;
4217
4218     mon_cmd($vmid, "set_link", name => $device,
4219                up => $up ? JSON::true : JSON::false);
4220 }
4221
4222 sub qemu_netdevadd {
4223     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4224
4225     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4226     my %options =  split(/[=,]/, $netdev);
4227
4228     if (defined(my $vhost = $options{vhost})) {
4229         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4230     }
4231
4232     if (defined(my $queues = $options{queues})) {
4233         $options{queues} = $queues + 0;
4234     }
4235
4236     mon_cmd($vmid, "netdev_add",  %options);
4237     return 1;
4238 }
4239
4240 sub qemu_netdevdel {
4241     my ($vmid, $deviceid) = @_;
4242
4243     mon_cmd($vmid, "netdev_del", id => $deviceid);
4244 }
4245
4246 sub qemu_usb_hotplug {
4247     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4248
4249     return if !$device;
4250
4251     # remove the old one first
4252     vm_deviceunplug($vmid, $conf, $deviceid);
4253
4254     # check if xhci controller is necessary and available
4255     if ($device->{usb3}) {
4256
4257         my $devicelist = vm_devices_list($vmid);
4258
4259         if (!$devicelist->{xhci}) {
4260             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4261             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4262         }
4263     }
4264     my $d = parse_usb_device($device->{host});
4265     $d->{usb3} = $device->{usb3};
4266
4267     # add the new one
4268     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4269 }
4270
4271 sub qemu_cpu_hotplug {
4272     my ($vmid, $conf, $vcpus) = @_;
4273
4274     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4275
4276     my $sockets = 1;
4277     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4278     $sockets = $conf->{sockets} if  $conf->{sockets};
4279     my $cores = $conf->{cores} || 1;
4280     my $maxcpus = $sockets * $cores;
4281
4282     $vcpus = $maxcpus if !$vcpus;
4283
4284     die "you can't add more vcpus than maxcpus\n"
4285         if $vcpus > $maxcpus;
4286
4287     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4288
4289     if ($vcpus < $currentvcpus) {
4290
4291         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4292
4293             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4294                 qemu_devicedel($vmid, "cpu$i");
4295                 my $retry = 0;
4296                 my $currentrunningvcpus = undef;
4297                 while (1) {
4298                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4299                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4300                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4301                     $retry++;
4302                     sleep 1;
4303                 }
4304                 #update conf after each succesfull cpu unplug
4305                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4306                 PVE::QemuConfig->write_config($vmid, $conf);
4307             }
4308         } else {
4309             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4310         }
4311
4312         return;
4313     }
4314
4315     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4316     die "vcpus in running vm does not match its configuration\n"
4317         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4318
4319     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4320
4321         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4322             my $cpustr = print_cpu_device($conf, $i);
4323             qemu_deviceadd($vmid, $cpustr);
4324
4325             my $retry = 0;
4326             my $currentrunningvcpus = undef;
4327             while (1) {
4328                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4329                 last if scalar(@{$currentrunningvcpus}) == $i;
4330                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4331                 sleep 1;
4332                 $retry++;
4333             }
4334             #update conf after each succesfull cpu hotplug
4335             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4336             PVE::QemuConfig->write_config($vmid, $conf);
4337         }
4338     } else {
4339
4340         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4341             mon_cmd($vmid, "cpu-add", id => int($i));
4342         }
4343     }
4344 }
4345
4346 sub qemu_block_set_io_throttle {
4347     my ($vmid, $deviceid,
4348         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4349         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4350         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4351         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4352
4353     return if !check_running($vmid) ;
4354
4355     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4356         bps => int($bps),
4357         bps_rd => int($bps_rd),
4358         bps_wr => int($bps_wr),
4359         iops => int($iops),
4360         iops_rd => int($iops_rd),
4361         iops_wr => int($iops_wr),
4362         bps_max => int($bps_max),
4363         bps_rd_max => int($bps_rd_max),
4364         bps_wr_max => int($bps_wr_max),
4365         iops_max => int($iops_max),
4366         iops_rd_max => int($iops_rd_max),
4367         iops_wr_max => int($iops_wr_max),
4368         bps_max_length => int($bps_max_length),
4369         bps_rd_max_length => int($bps_rd_max_length),
4370         bps_wr_max_length => int($bps_wr_max_length),
4371         iops_max_length => int($iops_max_length),
4372         iops_rd_max_length => int($iops_rd_max_length),
4373         iops_wr_max_length => int($iops_wr_max_length),
4374     );
4375
4376 }
4377
4378 sub qemu_block_resize {
4379     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4380
4381     my $running = check_running($vmid);
4382
4383     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4384
4385     return if !$running;
4386
4387     my $padding = (1024 - $size % 1024) % 1024;
4388     $size = $size + $padding;
4389
4390     mon_cmd(
4391         $vmid,
4392         "block_resize",
4393         device => $deviceid,
4394         size => int($size),
4395         timeout => 60,
4396     );
4397 }
4398
4399 sub qemu_volume_snapshot {
4400     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4401
4402     my $running = check_running($vmid);
4403
4404     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4405         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4406     } else {
4407         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4408     }
4409 }
4410
4411 sub qemu_volume_snapshot_delete {
4412     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4413
4414     my $running = check_running($vmid);
4415
4416     if($running) {
4417
4418         $running = undef;
4419         my $conf = PVE::QemuConfig->load_config($vmid);
4420         PVE::QemuConfig->foreach_volume($conf, sub {
4421             my ($ds, $drive) = @_;
4422             $running = 1 if $drive->{file} eq $volid;
4423         });
4424     }
4425
4426     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4427         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4428     } else {
4429         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4430     }
4431 }
4432
4433 sub set_migration_caps {
4434     my ($vmid, $savevm) = @_;
4435
4436     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4437
4438     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4439     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4440
4441     my $cap_ref = [];
4442
4443     my $enabled_cap = {
4444         "auto-converge" => 1,
4445         "xbzrle" => 1,
4446         "x-rdma-pin-all" => 0,
4447         "zero-blocks" => 0,
4448         "compress" => 0,
4449         "dirty-bitmaps" => $dirty_bitmaps,
4450     };
4451
4452     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4453
4454     for my $supported_capability (@$supported_capabilities) {
4455         push @$cap_ref, {
4456             capability => $supported_capability->{capability},
4457             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4458         };
4459     }
4460
4461     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4462 }
4463
4464 sub foreach_volid {
4465     my ($conf, $func, @param) = @_;
4466
4467     my $volhash = {};
4468
4469     my $test_volid = sub {
4470         my ($key, $drive, $snapname) = @_;
4471
4472         my $volid = $drive->{file};
4473         return if !$volid;
4474
4475         $volhash->{$volid}->{cdrom} //= 1;
4476         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4477
4478         my $replicate = $drive->{replicate} // 1;
4479         $volhash->{$volid}->{replicate} //= 0;
4480         $volhash->{$volid}->{replicate} = 1 if $replicate;
4481
4482         $volhash->{$volid}->{shared} //= 0;
4483         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4484
4485         $volhash->{$volid}->{referenced_in_config} //= 0;
4486         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4487
4488         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4489             if defined($snapname);
4490
4491         my $size = $drive->{size};
4492         $volhash->{$volid}->{size} //= $size if $size;
4493
4494         $volhash->{$volid}->{is_vmstate} //= 0;
4495         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4496
4497         $volhash->{$volid}->{is_unused} //= 0;
4498         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4499
4500         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4501     };
4502
4503     my $include_opts = {
4504         extra_keys => ['vmstate'],
4505         include_unused => 1,
4506     };
4507
4508     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4509     foreach my $snapname (keys %{$conf->{snapshots}}) {
4510         my $snap = $conf->{snapshots}->{$snapname};
4511         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4512     }
4513
4514     foreach my $volid (keys %$volhash) {
4515         &$func($volid, $volhash->{$volid}, @param);
4516     }
4517 }
4518
4519 my $fast_plug_option = {
4520     'lock' => 1,
4521     'name' => 1,
4522     'onboot' => 1,
4523     'shares' => 1,
4524     'startup' => 1,
4525     'description' => 1,
4526     'protection' => 1,
4527     'vmstatestorage' => 1,
4528     'hookscript' => 1,
4529     'tags' => 1,
4530 };
4531
4532 # hotplug changes in [PENDING]
4533 # $selection hash can be used to only apply specified options, for
4534 # example: { cores => 1 } (only apply changed 'cores')
4535 # $errors ref is used to return error messages
4536 sub vmconfig_hotplug_pending {
4537     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4538
4539     my $defaults = load_defaults();
4540     my $arch = get_vm_arch($conf);
4541     my $machine_type = get_vm_machine($conf, undef, $arch);
4542
4543     # commit values which do not have any impact on running VM first
4544     # Note: those option cannot raise errors, we we do not care about
4545     # $selection and always apply them.
4546
4547     my $add_error = sub {
4548         my ($opt, $msg) = @_;
4549         $errors->{$opt} = "hotplug problem - $msg";
4550     };
4551
4552     my $changes = 0;
4553     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4554         if ($fast_plug_option->{$opt}) {
4555             $conf->{$opt} = $conf->{pending}->{$opt};
4556             delete $conf->{pending}->{$opt};
4557             $changes = 1;
4558         }
4559     }
4560
4561     if ($changes) {
4562         PVE::QemuConfig->write_config($vmid, $conf);
4563     }
4564
4565     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4566
4567     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4568     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4569     foreach my $opt (sort keys %$pending_delete_hash) {
4570         next if $selection && !$selection->{$opt};
4571         my $force = $pending_delete_hash->{$opt}->{force};
4572         eval {
4573             if ($opt eq 'hotplug') {
4574                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4575             } elsif ($opt eq 'tablet') {
4576                 die "skip\n" if !$hotplug_features->{usb};
4577                 if ($defaults->{tablet}) {
4578                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4579                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4580                         if $arch eq 'aarch64';
4581                 } else {
4582                     vm_deviceunplug($vmid, $conf, 'tablet');
4583                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4584                 }
4585             } elsif ($opt =~ m/^usb\d+/) {
4586                 die "skip\n";
4587                 # since we cannot reliably hot unplug usb devices we are disabling it
4588                 #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4589                 #vm_deviceunplug($vmid, $conf, $opt);
4590             } elsif ($opt eq 'vcpus') {
4591                 die "skip\n" if !$hotplug_features->{cpu};
4592                 qemu_cpu_hotplug($vmid, $conf, undef);
4593             } elsif ($opt eq 'balloon') {
4594                 # enable balloon device is not hotpluggable
4595                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4596                 # here we reset the ballooning value to memory
4597                 my $balloon = $conf->{memory} || $defaults->{memory};
4598                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4599             } elsif ($fast_plug_option->{$opt}) {
4600                 # do nothing
4601             } elsif ($opt =~ m/^net(\d+)$/) {
4602                 die "skip\n" if !$hotplug_features->{network};
4603                 vm_deviceunplug($vmid, $conf, $opt);
4604             } elsif (is_valid_drivename($opt)) {
4605                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4606                 vm_deviceunplug($vmid, $conf, $opt);
4607                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4608             } elsif ($opt =~ m/^memory$/) {
4609                 die "skip\n" if !$hotplug_features->{memory};
4610                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4611             } elsif ($opt eq 'cpuunits') {
4612                 $cgroup->change_cpu_shares(undef, $defaults->{cpuunits});
4613             } elsif ($opt eq 'cpulimit') {
4614                 $cgroup->change_cpu_quota(-1, 100000);
4615             } else {
4616                 die "skip\n";
4617             }
4618         };
4619         if (my $err = $@) {
4620             &$add_error($opt, $err) if $err ne "skip\n";
4621         } else {
4622             delete $conf->{$opt};
4623             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4624         }
4625     }
4626
4627     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4628     $apply_pending_cloudinit = sub {
4629         return if $apply_pending_cloudinit_done; # once is enough
4630         $apply_pending_cloudinit_done = 1; # once is enough
4631
4632         my ($key, $value) = @_;
4633
4634         my @cloudinit_opts = keys %$confdesc_cloudinit;
4635         foreach my $opt (keys %{$conf->{pending}}) {
4636             next if !grep { $_ eq $opt } @cloudinit_opts;
4637             $conf->{$opt} = delete $conf->{pending}->{$opt};
4638         }
4639
4640         my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4641         foreach my $opt (sort keys %$pending_delete_hash) {
4642             next if !grep { $_ eq $opt } @cloudinit_opts;
4643             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4644             delete $conf->{$opt};
4645         }
4646
4647         my $new_conf = { %$conf };
4648         $new_conf->{$key} = $value;
4649         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4650     };
4651
4652     foreach my $opt (keys %{$conf->{pending}}) {
4653         next if $selection && !$selection->{$opt};
4654         my $value = $conf->{pending}->{$opt};
4655         eval {
4656             if ($opt eq 'hotplug') {
4657                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4658             } elsif ($opt eq 'tablet') {
4659                 die "skip\n" if !$hotplug_features->{usb};
4660                 if ($value == 1) {
4661                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4662                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4663                         if $arch eq 'aarch64';
4664                 } elsif ($value == 0) {
4665                     vm_deviceunplug($vmid, $conf, 'tablet');
4666                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4667                 }
4668             } elsif ($opt =~ m/^usb\d+$/) {
4669                 die "skip\n";
4670                 # since we cannot reliably hot unplug usb devices we disable it for now
4671                 #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4672                 #my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4673                 #die "skip\n" if !$d;
4674                 #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4675             } elsif ($opt eq 'vcpus') {
4676                 die "skip\n" if !$hotplug_features->{cpu};
4677                 qemu_cpu_hotplug($vmid, $conf, $value);
4678             } elsif ($opt eq 'balloon') {
4679                 # enable/disable balloning device is not hotpluggable
4680                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4681                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4682                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4683
4684                 # allow manual ballooning if shares is set to zero
4685                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4686                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4687                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4688                 }
4689             } elsif ($opt =~ m/^net(\d+)$/) {
4690                 # some changes can be done without hotplug
4691                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4692                                     $vmid, $opt, $value, $arch, $machine_type);
4693             } elsif (is_valid_drivename($opt)) {
4694                 die "skip\n" if $opt eq 'efidisk0';
4695                 # some changes can be done without hotplug
4696                 my $drive = parse_drive($opt, $value);
4697                 if (drive_is_cloudinit($drive)) {
4698                     &$apply_pending_cloudinit($opt, $value);
4699                 }
4700                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4701                                      $vmid, $opt, $value, $arch, $machine_type);
4702             } elsif ($opt =~ m/^memory$/) { #dimms
4703                 die "skip\n" if !$hotplug_features->{memory};
4704                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4705             } elsif ($opt eq 'cpuunits') {
4706                 $cgroup->change_cpu_shares($conf->{pending}->{$opt}, $defaults->{cpuunits});
4707             } elsif ($opt eq 'cpulimit') {
4708                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4709                 $cgroup->change_cpu_quota($cpulimit, 100000);
4710             } else {
4711                 die "skip\n";  # skip non-hot-pluggable options
4712             }
4713         };
4714         if (my $err = $@) {
4715             &$add_error($opt, $err) if $err ne "skip\n";
4716         } else {
4717             $conf->{$opt} = $value;
4718             delete $conf->{pending}->{$opt};
4719         }
4720     }
4721
4722     PVE::QemuConfig->write_config($vmid, $conf);
4723 }
4724
4725 sub try_deallocate_drive {
4726     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4727
4728     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4729         my $volid = $drive->{file};
4730         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4731             my $sid = PVE::Storage::parse_volume_id($volid);
4732             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4733
4734             # check if the disk is really unused
4735             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4736                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4737             PVE::Storage::vdisk_free($storecfg, $volid);
4738             return 1;
4739         } else {
4740             # If vm is not owner of this disk remove from config
4741             return 1;
4742         }
4743     }
4744
4745     return;
4746 }
4747
4748 sub vmconfig_delete_or_detach_drive {
4749     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4750
4751     my $drive = parse_drive($opt, $conf->{$opt});
4752
4753     my $rpcenv = PVE::RPCEnvironment::get();
4754     my $authuser = $rpcenv->get_user();
4755
4756     if ($force) {
4757         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4758         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4759     } else {
4760         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4761     }
4762 }
4763
4764
4765
4766 sub vmconfig_apply_pending {
4767     my ($vmid, $conf, $storecfg, $errors) = @_;
4768
4769     my $add_apply_error = sub {
4770         my ($opt, $msg) = @_;
4771         my $err_msg = "unable to apply pending change $opt : $msg";
4772         $errors->{$opt} = $err_msg;
4773         warn $err_msg;
4774     };
4775
4776     # cold plug
4777
4778     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4779     foreach my $opt (sort keys %$pending_delete_hash) {
4780         my $force = $pending_delete_hash->{$opt}->{force};
4781         eval {
4782             if ($opt =~ m/^unused/) {
4783                 die "internal error";
4784             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4785                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4786             }
4787         };
4788         if (my $err = $@) {
4789             $add_apply_error->($opt, $err);
4790         } else {
4791             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4792             delete $conf->{$opt};
4793         }
4794     }
4795
4796     PVE::QemuConfig->cleanup_pending($conf);
4797
4798     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4799         next if $opt eq 'delete'; # just to be sure
4800         eval {
4801             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4802                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4803             }
4804         };
4805         if (my $err = $@) {
4806             $add_apply_error->($opt, $err);
4807         } else {
4808             $conf->{$opt} = delete $conf->{pending}->{$opt};
4809         }
4810     }
4811
4812     # write all changes at once to avoid unnecessary i/o
4813     PVE::QemuConfig->write_config($vmid, $conf);
4814 }
4815
4816 sub vmconfig_update_net {
4817     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4818
4819     my $newnet = parse_net($value);
4820
4821     if ($conf->{$opt}) {
4822         my $oldnet = parse_net($conf->{$opt});
4823
4824         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4825             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4826             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4827             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4828
4829             # for non online change, we try to hot-unplug
4830             die "skip\n" if !$hotplug;
4831             vm_deviceunplug($vmid, $conf, $opt);
4832         } else {
4833
4834             die "internal error" if $opt !~ m/net(\d+)/;
4835             my $iface = "tap${vmid}i$1";
4836
4837             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4838                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4839                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4840                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4841                 PVE::Network::tap_unplug($iface);
4842
4843                 if ($have_sdn) {
4844                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4845                 } else {
4846                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4847                 }
4848             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4849                 # Rate can be applied on its own but any change above needs to
4850                 # include the rate in tap_plug since OVS resets everything.
4851                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4852             }
4853
4854             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4855                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4856             }
4857
4858             return 1;
4859         }
4860     }
4861
4862     if ($hotplug) {
4863         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4864     } else {
4865         die "skip\n";
4866     }
4867 }
4868
4869 sub vmconfig_update_disk {
4870     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4871
4872     my $drive = parse_drive($opt, $value);
4873
4874     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
4875         my $media = $drive->{media} || 'disk';
4876         my $oldmedia = $old_drive->{media} || 'disk';
4877         die "unable to change media type\n" if $media ne $oldmedia;
4878
4879         if (!drive_is_cdrom($old_drive)) {
4880
4881             if ($drive->{file} ne $old_drive->{file}) {
4882
4883                 die "skip\n" if !$hotplug;
4884
4885                 # unplug and register as unused
4886                 vm_deviceunplug($vmid, $conf, $opt);
4887                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4888
4889             } else {
4890                 # update existing disk
4891
4892                 # skip non hotpluggable value
4893                 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4894                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4895                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4896                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
4897                     safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
4898                     die "skip\n";
4899                 }
4900
4901                 # apply throttle
4902                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4903                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4904                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4905                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4906                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4907                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4908                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4909                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4910                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4911                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4912                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4913                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4914                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4915                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4916                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4917                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4918                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4919                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4920
4921                     qemu_block_set_io_throttle(
4922                         $vmid,"drive-$opt",
4923                         ($drive->{mbps} || 0)*1024*1024,
4924                         ($drive->{mbps_rd} || 0)*1024*1024,
4925                         ($drive->{mbps_wr} || 0)*1024*1024,
4926                          $drive->{iops} || 0,
4927                          $drive->{iops_rd} || 0,
4928                          $drive->{iops_wr} || 0,
4929                         ($drive->{mbps_max} || 0)*1024*1024,
4930                         ($drive->{mbps_rd_max} || 0)*1024*1024,
4931                         ($drive->{mbps_wr_max} || 0)*1024*1024,
4932                          $drive->{iops_max} || 0,
4933                          $drive->{iops_rd_max} || 0,
4934                          $drive->{iops_wr_max} || 0,
4935                          $drive->{bps_max_length} || 1,
4936                          $drive->{bps_rd_max_length} || 1,
4937                          $drive->{bps_wr_max_length} || 1,
4938                          $drive->{iops_max_length} || 1,
4939                          $drive->{iops_rd_max_length} || 1,
4940                          $drive->{iops_wr_max_length} || 1,
4941                     );
4942
4943                 }
4944
4945                 return 1;
4946             }
4947
4948         } else { # cdrom
4949
4950             if ($drive->{file} eq 'none') {
4951                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4952                 if (drive_is_cloudinit($old_drive)) {
4953                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4954                 }
4955             } else {
4956                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4957
4958                 # force eject if locked
4959                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4960
4961                 if ($path) {
4962                     mon_cmd($vmid, "blockdev-change-medium",
4963                         id => "$opt", filename => "$path");
4964                 }
4965             }
4966
4967             return 1;
4968         }
4969     }
4970
4971     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4972     # hotplug new disks
4973     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4974     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
4975 }
4976
4977 # called in locked context by incoming migration
4978 sub vm_migrate_get_nbd_disks {
4979     my ($storecfg, $conf, $replicated_volumes) = @_;
4980
4981     my $local_volumes = {};
4982     PVE::QemuConfig->foreach_volume($conf, sub {
4983         my ($ds, $drive) = @_;
4984
4985         return if drive_is_cdrom($drive);
4986
4987         my $volid = $drive->{file};
4988
4989         return if !$volid;
4990
4991         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4992
4993         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4994         return if $scfg->{shared};
4995
4996         # replicated disks re-use existing state via bitmap
4997         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
4998         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
4999     });
5000     return $local_volumes;
5001 }
5002
5003 # called in locked context by incoming migration
5004 sub vm_migrate_alloc_nbd_disks {
5005     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5006
5007     my $format = undef;
5008
5009     my $nbd = {};
5010     foreach my $opt (sort keys %$source_volumes) {
5011         my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
5012
5013         if ($use_existing) {
5014             $nbd->{$opt}->{drivestr} = print_drive($drive);
5015             $nbd->{$opt}->{volid} = $volid;
5016             $nbd->{$opt}->{replicated} = 1;
5017             next;
5018         }
5019
5020         # If a remote storage is specified and the format of the original
5021         # volume is not available there, fall back to the default format.
5022         # Otherwise use the same format as the original.
5023         if (!$storagemap->{identity}) {
5024             $storeid = map_storage($storagemap, $storeid);
5025             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5026             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5027             my $fileFormat = qemu_img_format($scfg, $volname);
5028             $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
5029         } else {
5030             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5031             $format = qemu_img_format($scfg, $volname);
5032         }
5033
5034         my $size = $drive->{size} / 1024;
5035         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5036         my $newdrive = $drive;
5037         $newdrive->{format} = $format;
5038         $newdrive->{file} = $newvolid;
5039         my $drivestr = print_drive($newdrive);
5040         $nbd->{$opt}->{drivestr} = $drivestr;
5041         $nbd->{$opt}->{volid} = $newvolid;
5042     }
5043
5044     return $nbd;
5045 }
5046
5047 # see vm_start_nolock for parameters, additionally:
5048 # migrate_opts:
5049 #   storagemap = parsed storage map for allocating NBD disks
5050 sub vm_start {
5051     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5052
5053     return PVE::QemuConfig->lock_config($vmid, sub {
5054         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5055
5056         die "you can't start a vm if it's a template\n"
5057             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5058
5059         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5060         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5061
5062         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5063
5064         if ($has_backup_lock && $running) {
5065             # a backup is currently running, attempt to start the guest in the
5066             # existing QEMU instance
5067             return vm_resume($vmid);
5068         }
5069
5070         PVE::QemuConfig->check_lock($conf)
5071             if !($params->{skiplock} || $has_suspended_lock);
5072
5073         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5074
5075         die "VM $vmid already running\n" if $running;
5076
5077         if (my $storagemap = $migrate_opts->{storagemap}) {
5078             my $replicated = $migrate_opts->{replicated_volumes};
5079             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5080             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5081
5082             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5083                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5084             }
5085         }
5086
5087         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5088     });
5089 }
5090
5091
5092 # params:
5093 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5094 #   skiplock => 0/1, skip checking for config lock
5095 #   skiptemplate => 0/1, skip checking whether VM is template
5096 #   forcemachine => to force Qemu machine (rollback/migration)
5097 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5098 #   timeout => in seconds
5099 #   paused => start VM in paused state (backup)
5100 #   resume => resume from hibernation
5101 #   pbs-backing => {
5102 #      sata0 => {
5103 #         repository
5104 #         snapshot
5105 #         keyfile
5106 #         archive
5107 #      },
5108 #      virtio2 => ...
5109 #   }
5110 # migrate_opts:
5111 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5112 #   migratedfrom => source node
5113 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5114 #   network => CIDR of migration network
5115 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5116 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5117 #   replicated_volumes = which volids should be re-used with bitmaps for nbd migration
5118 sub vm_start_nolock {
5119     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5120
5121     my $statefile = $params->{statefile};
5122     my $resume = $params->{resume};
5123
5124     my $migratedfrom = $migrate_opts->{migratedfrom};
5125     my $migration_type = $migrate_opts->{type};
5126
5127     my $res = {};
5128
5129     # clean up leftover reboot request files
5130     eval { clear_reboot_request($vmid); };
5131     warn $@ if $@;
5132
5133     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5134         vmconfig_apply_pending($vmid, $conf, $storecfg);
5135         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5136     }
5137
5138     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5139
5140     my $defaults = load_defaults();
5141
5142     # set environment variable useful inside network script
5143     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5144
5145     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5146
5147     my $forcemachine = $params->{forcemachine};
5148     my $forcecpu = $params->{forcecpu};
5149     if ($resume) {
5150         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5151         $forcemachine = $conf->{runningmachine};
5152         $forcecpu = $conf->{runningcpu};
5153         print "Resuming suspended VM\n";
5154     }
5155
5156     my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid,
5157         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5158
5159     my $migration_ip;
5160     my $get_migration_ip = sub {
5161         my ($nodename) = @_;
5162
5163         return $migration_ip if defined($migration_ip);
5164
5165         my $cidr = $migrate_opts->{network};
5166
5167         if (!defined($cidr)) {
5168             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5169             $cidr = $dc_conf->{migration}->{network};
5170         }
5171
5172         if (defined($cidr)) {
5173             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5174
5175             die "could not get IP: no address configured on local " .
5176                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5177
5178             die "could not get IP: multiple addresses configured on local " .
5179                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5180
5181             $migration_ip = @$ips[0];
5182         }
5183
5184         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5185             if !defined($migration_ip);
5186
5187         return $migration_ip;
5188     };
5189
5190     my $migrate_uri;
5191     if ($statefile) {
5192         if ($statefile eq 'tcp') {
5193             my $localip = "localhost";
5194             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5195             my $nodename = nodename();
5196
5197             if (!defined($migration_type)) {
5198                 if (defined($datacenterconf->{migration}->{type})) {
5199                     $migration_type = $datacenterconf->{migration}->{type};
5200                 } else {
5201                     $migration_type = 'secure';
5202                 }
5203             }
5204
5205             if ($migration_type eq 'insecure') {
5206                 $localip = $get_migration_ip->($nodename);
5207                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5208             }
5209
5210             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5211             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5212             $migrate_uri = "tcp:${localip}:${migrate_port}";
5213             push @$cmd, '-incoming', $migrate_uri;
5214             push @$cmd, '-S';
5215
5216         } elsif ($statefile eq 'unix') {
5217             # should be default for secure migrations as a ssh TCP forward
5218             # tunnel is not deterministic reliable ready and fails regurarly
5219             # to set up in time, so use UNIX socket forwards
5220             my $socket_addr = "/run/qemu-server/$vmid.migrate";
5221             unlink $socket_addr;
5222
5223             $migrate_uri = "unix:$socket_addr";
5224
5225             push @$cmd, '-incoming', $migrate_uri;
5226             push @$cmd, '-S';
5227
5228         } elsif (-e $statefile) {
5229             push @$cmd, '-loadstate', $statefile;
5230         } else {
5231             my $statepath = PVE::Storage::path($storecfg, $statefile);
5232             push @$vollist, $statefile;
5233             push @$cmd, '-loadstate', $statepath;
5234         }
5235     } elsif ($params->{paused}) {
5236         push @$cmd, '-S';
5237     }
5238
5239     # host pci devices
5240     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5241       my $d = parse_hostpci($conf->{"hostpci$i"});
5242       next if !$d;
5243       my $pcidevices = $d->{pciid};
5244       foreach my $pcidevice (@$pcidevices) {
5245             my $pciid = $pcidevice->{id};
5246
5247             my $info = PVE::SysFSTools::pci_device_info("$pciid");
5248             die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5249             die "no pci device info for device '$pciid'\n" if !$info;
5250
5251             if ($d->{mdev}) {
5252                 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5253                 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5254             } else {
5255                 die "can't unbind/bind PCI group to VFIO '$pciid'\n"
5256                     if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5257                 die "can't reset PCI device '$pciid'\n"
5258                     if $info->{has_fl_reset} && !PVE::SysFSTools::pci_dev_reset($info);
5259             }
5260       }
5261     }
5262
5263     PVE::Storage::activate_volumes($storecfg, $vollist);
5264
5265     eval {
5266         run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5267             outfunc => sub {}, errfunc => sub {});
5268     };
5269     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5270     # timeout should be more than enough here...
5271     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5272
5273     my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5274                                               : $defaults->{cpuunits};
5275
5276     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5277     my %run_params = (
5278         timeout => $statefile ? undef : $start_timeout,
5279         umask => 0077,
5280         noerr => 1,
5281     );
5282
5283     # when migrating, prefix QEMU output so other side can pick up any
5284     # errors that might occur and show the user
5285     if ($migratedfrom) {
5286         $run_params{quiet} = 1;
5287         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5288     }
5289
5290     my %properties = (
5291         Slice => 'qemu.slice',
5292         KillMode => 'process',
5293         SendSIGKILL => 0,
5294         TimeoutStopUSec => ULONG_MAX, # infinity
5295     );
5296
5297     if (PVE::CGroup::cgroup_mode() == 2) {
5298         $properties{CPUWeight} = $cpuunits;
5299     } else {
5300         $properties{CPUShares} = $cpuunits;
5301     }
5302
5303     if (my $cpulimit = $conf->{cpulimit}) {
5304         $properties{CPUQuota} = int($cpulimit * 100);
5305     }
5306     $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5307
5308     my $run_qemu = sub {
5309         PVE::Tools::run_fork sub {
5310             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5311
5312             my $exitcode = run_command($cmd, %run_params);
5313             die "QEMU exited with code $exitcode\n" if $exitcode;
5314         };
5315     };
5316
5317     if ($conf->{hugepages}) {
5318
5319         my $code = sub {
5320             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5321             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5322
5323             PVE::QemuServer::Memory::hugepages_mount();
5324             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5325
5326             eval { $run_qemu->() };
5327             if (my $err = $@) {
5328                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5329                     if !$conf->{keephugepages};
5330                 die $err;
5331             }
5332
5333             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5334                 if !$conf->{keephugepages};
5335         };
5336         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5337
5338     } else {
5339         eval { $run_qemu->() };
5340     }
5341
5342     if (my $err = $@) {
5343         # deactivate volumes if start fails
5344         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5345         die "start failed: $err";
5346     }
5347
5348     print "migration listens on $migrate_uri\n" if $migrate_uri;
5349     $res->{migrate_uri} = $migrate_uri;
5350
5351     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5352         eval { mon_cmd($vmid, "cont"); };
5353         warn $@ if $@;
5354     }
5355
5356     #start nbd server for storage migration
5357     if (my $nbd = $migrate_opts->{nbd}) {
5358         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5359
5360         my $migrate_storage_uri;
5361         # nbd_protocol_version > 0 for unix socket support
5362         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5363             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5364             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5365             $migrate_storage_uri = "nbd:unix:$socket_path";
5366         } else {
5367             my $nodename = nodename();
5368             my $localip = $get_migration_ip->($nodename);
5369             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5370             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5371
5372             mon_cmd($vmid, "nbd-server-start", addr => {
5373                 type => 'inet',
5374                 data => {
5375                     host => "${localip}",
5376                     port => "${storage_migrate_port}",
5377                 },
5378             });
5379             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5380             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5381         }
5382
5383         $res->{migrate_storage_uri} = $migrate_storage_uri;
5384
5385         foreach my $opt (sort keys %$nbd) {
5386             my $drivestr = $nbd->{$opt}->{drivestr};
5387             my $volid = $nbd->{$opt}->{volid};
5388             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5389             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5390             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5391             print "re-using replicated volume: $opt - $volid\n"
5392                 if $nbd->{$opt}->{replicated};
5393
5394             $res->{drives}->{$opt} = $nbd->{$opt};
5395             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5396         }
5397     }
5398
5399     if ($migratedfrom) {
5400         eval {
5401             set_migration_caps($vmid);
5402         };
5403         warn $@ if $@;
5404
5405         if ($spice_port) {
5406             print "spice listens on port $spice_port\n";
5407             $res->{spice_port} = $spice_port;
5408             if ($migrate_opts->{spice_ticket}) {
5409                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5410                     $migrate_opts->{spice_ticket});
5411                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5412             }
5413         }
5414
5415     } else {
5416         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5417             if !$statefile && $conf->{balloon};
5418
5419         foreach my $opt (keys %$conf) {
5420             next if $opt !~  m/^net\d+$/;
5421             my $nicconf = parse_net($conf->{$opt});
5422             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5423         }
5424     }
5425
5426     mon_cmd($vmid, 'qom-set',
5427                 path => "machine/peripheral/balloon0",
5428                 property => "guest-stats-polling-interval",
5429                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5430
5431     if ($resume) {
5432         print "Resumed VM, removing state\n";
5433         if (my $vmstate = $conf->{vmstate}) {
5434             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5435             PVE::Storage::vdisk_free($storecfg, $vmstate);
5436         }
5437         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5438         PVE::QemuConfig->write_config($vmid, $conf);
5439     }
5440
5441     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5442
5443     return $res;
5444 }
5445
5446 sub vm_commandline {
5447     my ($storecfg, $vmid, $snapname) = @_;
5448
5449     my $conf = PVE::QemuConfig->load_config($vmid);
5450     my $forcemachine;
5451     my $forcecpu;
5452
5453     if ($snapname) {
5454         my $snapshot = $conf->{snapshots}->{$snapname};
5455         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5456
5457         # check for machine or CPU overrides in snapshot
5458         $forcemachine = $snapshot->{runningmachine};
5459         $forcecpu = $snapshot->{runningcpu};
5460
5461         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5462
5463         $conf = $snapshot;
5464     }
5465
5466     my $defaults = load_defaults();
5467
5468     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults,
5469         $forcemachine, $forcecpu);
5470
5471     return PVE::Tools::cmd2string($cmd);
5472 }
5473
5474 sub vm_reset {
5475     my ($vmid, $skiplock) = @_;
5476
5477     PVE::QemuConfig->lock_config($vmid, sub {
5478
5479         my $conf = PVE::QemuConfig->load_config($vmid);
5480
5481         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5482
5483         mon_cmd($vmid, "system_reset");
5484     });
5485 }
5486
5487 sub get_vm_volumes {
5488     my ($conf) = @_;
5489
5490     my $vollist = [];
5491     foreach_volid($conf, sub {
5492         my ($volid, $attr) = @_;
5493
5494         return if $volid =~ m|^/|;
5495
5496         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5497         return if !$sid;
5498
5499         push @$vollist, $volid;
5500     });
5501
5502     return $vollist;
5503 }
5504
5505 sub vm_stop_cleanup {
5506     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5507
5508     eval {
5509
5510         if (!$keepActive) {
5511             my $vollist = get_vm_volumes($conf);
5512             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5513         }
5514
5515         foreach my $ext (qw(mon qmp pid vnc qga)) {
5516             unlink "/var/run/qemu-server/${vmid}.$ext";
5517         }
5518
5519         if ($conf->{ivshmem}) {
5520             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5521             # just delete it for now, VMs which have this already open do not
5522             # are affected, but new VMs will get a separated one. If this
5523             # becomes an issue we either add some sort of ref-counting or just
5524             # add a "don't delete on stop" flag to the ivshmem format.
5525             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5526         }
5527
5528         foreach my $key (keys %$conf) {
5529             next if $key !~ m/^hostpci(\d+)$/;
5530             my $hostpciindex = $1;
5531             my $d = parse_hostpci($conf->{$key});
5532             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5533
5534             foreach my $pci (@{$d->{pciid}}) {
5535                 my $pciid = $pci->{id};
5536                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5537             }
5538         }
5539
5540         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5541     };
5542     warn $@ if $@; # avoid errors - just warn
5543 }
5544
5545 # call only in locked context
5546 sub _do_vm_stop {
5547     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5548
5549     my $pid = check_running($vmid, $nocheck);
5550     return if !$pid;
5551
5552     my $conf;
5553     if (!$nocheck) {
5554         $conf = PVE::QemuConfig->load_config($vmid);
5555         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5556         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5557             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5558             $timeout = $opts->{down} if $opts->{down};
5559         }
5560         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5561     }
5562
5563     eval {
5564         if ($shutdown) {
5565             if (defined($conf) && get_qga_key($conf, 'enabled')) {
5566                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5567             } else {
5568                 mon_cmd($vmid, "system_powerdown");
5569             }
5570         } else {
5571             mon_cmd($vmid, "quit");
5572         }
5573     };
5574     my $err = $@;
5575
5576     if (!$err) {
5577         $timeout = 60 if !defined($timeout);
5578
5579         my $count = 0;
5580         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5581             $count++;
5582             sleep 1;
5583         }
5584
5585         if ($count >= $timeout) {
5586             if ($force) {
5587                 warn "VM still running - terminating now with SIGTERM\n";
5588                 kill 15, $pid;
5589             } else {
5590                 die "VM quit/powerdown failed - got timeout\n";
5591             }
5592         } else {
5593             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5594             return;
5595         }
5596     } else {
5597        if (!check_running($vmid, $nocheck)) {
5598             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5599             return;
5600        }
5601        if ($force) {
5602             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5603             kill 15, $pid;
5604         } else {
5605             die "VM quit/powerdown failed\n";
5606         }
5607     }
5608
5609     # wait again
5610     $timeout = 10;
5611
5612     my $count = 0;
5613     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5614         $count++;
5615         sleep 1;
5616     }
5617
5618     if ($count >= $timeout) {
5619         warn "VM still running - terminating now with SIGKILL\n";
5620         kill 9, $pid;
5621         sleep 1;
5622     }
5623
5624     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5625 }
5626
5627 # Note: use $nocheck to skip tests if VM configuration file exists.
5628 # We need that when migration VMs to other nodes (files already moved)
5629 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5630 sub vm_stop {
5631     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5632
5633     $force = 1 if !defined($force) && !$shutdown;
5634
5635     if ($migratedfrom){
5636         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5637         kill 15, $pid if $pid;
5638         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5639         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5640         return;
5641     }
5642
5643     PVE::QemuConfig->lock_config($vmid, sub {
5644         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5645    });
5646 }
5647
5648 sub vm_reboot {
5649     my ($vmid, $timeout) = @_;
5650
5651     PVE::QemuConfig->lock_config($vmid, sub {
5652         eval {
5653
5654             # only reboot if running, as qmeventd starts it again on a stop event
5655             return if !check_running($vmid);
5656
5657             create_reboot_request($vmid);
5658
5659             my $storecfg = PVE::Storage::config();
5660             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5661
5662         };
5663         if (my $err = $@) {
5664             # avoid that the next normal shutdown will be confused for a reboot
5665             clear_reboot_request($vmid);
5666             die $err;
5667         }
5668    });
5669 }
5670
5671 # note: if using the statestorage parameter, the caller has to check privileges
5672 sub vm_suspend {
5673     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5674
5675     my $conf;
5676     my $path;
5677     my $storecfg;
5678     my $vmstate;
5679
5680     PVE::QemuConfig->lock_config($vmid, sub {
5681
5682         $conf = PVE::QemuConfig->load_config($vmid);
5683
5684         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5685         PVE::QemuConfig->check_lock($conf)
5686             if !($skiplock || $is_backing_up);
5687
5688         die "cannot suspend to disk during backup\n"
5689             if $is_backing_up && $includestate;
5690
5691         if ($includestate) {
5692             $conf->{lock} = 'suspending';
5693             my $date = strftime("%Y-%m-%d", localtime(time()));
5694             $storecfg = PVE::Storage::config();
5695             if (!$statestorage) {
5696                 $statestorage = find_vmstate_storage($conf, $storecfg);
5697                 # check permissions for the storage
5698                 my $rpcenv = PVE::RPCEnvironment::get();
5699                 if ($rpcenv->{type} ne 'cli') {
5700                     my $authuser = $rpcenv->get_user();
5701                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5702                 }
5703             }
5704
5705
5706             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
5707                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5708             $path = PVE::Storage::path($storecfg, $vmstate);
5709             PVE::QemuConfig->write_config($vmid, $conf);
5710         } else {
5711             mon_cmd($vmid, "stop");
5712         }
5713     });
5714
5715     if ($includestate) {
5716         # save vm state
5717         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5718
5719         eval {
5720             set_migration_caps($vmid, 1);
5721             mon_cmd($vmid, "savevm-start", statefile => $path);
5722             for(;;) {
5723                 my $state = mon_cmd($vmid, "query-savevm");
5724                 if (!$state->{status}) {
5725                     die "savevm not active\n";
5726                 } elsif ($state->{status} eq 'active') {
5727                     sleep(1);
5728                     next;
5729                 } elsif ($state->{status} eq 'completed') {
5730                     print "State saved, quitting\n";
5731                     last;
5732                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5733                     die "query-savevm failed with error '$state->{error}'\n"
5734                 } else {
5735                     die "query-savevm returned status '$state->{status}'\n";
5736                 }
5737             }
5738         };
5739         my $err = $@;
5740
5741         PVE::QemuConfig->lock_config($vmid, sub {
5742             $conf = PVE::QemuConfig->load_config($vmid);
5743             if ($err) {
5744                 # cleanup, but leave suspending lock, to indicate something went wrong
5745                 eval {
5746                     mon_cmd($vmid, "savevm-end");
5747                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5748                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5749                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
5750                     PVE::QemuConfig->write_config($vmid, $conf);
5751                 };
5752                 warn $@ if $@;
5753                 die $err;
5754             }
5755
5756             die "lock changed unexpectedly\n"
5757                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5758
5759             mon_cmd($vmid, "quit");
5760             $conf->{lock} = 'suspended';
5761             PVE::QemuConfig->write_config($vmid, $conf);
5762         });
5763     }
5764 }
5765
5766 sub vm_resume {
5767     my ($vmid, $skiplock, $nocheck) = @_;
5768
5769     PVE::QemuConfig->lock_config($vmid, sub {
5770         my $res = mon_cmd($vmid, 'query-status');
5771         my $resume_cmd = 'cont';
5772         my $reset = 0;
5773
5774         if ($res->{status}) {
5775             return if $res->{status} eq 'running'; # job done, go home
5776             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
5777             $reset = 1 if $res->{status} eq 'shutdown';
5778         }
5779
5780         if (!$nocheck) {
5781
5782             my $conf = PVE::QemuConfig->load_config($vmid);
5783
5784             PVE::QemuConfig->check_lock($conf)
5785                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5786         }
5787
5788         if ($reset) {
5789             # required if a VM shuts down during a backup and we get a resume
5790             # request before the backup finishes for example
5791             mon_cmd($vmid, "system_reset");
5792         }
5793         mon_cmd($vmid, $resume_cmd);
5794     });
5795 }
5796
5797 sub vm_sendkey {
5798     my ($vmid, $skiplock, $key) = @_;
5799
5800     PVE::QemuConfig->lock_config($vmid, sub {
5801
5802         my $conf = PVE::QemuConfig->load_config($vmid);
5803
5804         # there is no qmp command, so we use the human monitor command
5805         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5806         die $res if $res ne '';
5807     });
5808 }
5809
5810 # vzdump restore implementaion
5811
5812 sub tar_archive_read_firstfile {
5813     my $archive = shift;
5814
5815     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5816
5817     # try to detect archive type first
5818     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5819         die "unable to open file '$archive'\n";
5820     my $firstfile = <$fh>;
5821     kill 15, $pid;
5822     close $fh;
5823
5824     die "ERROR: archive contaions no data\n" if !$firstfile;
5825     chomp $firstfile;
5826
5827     return $firstfile;
5828 }
5829
5830 sub tar_restore_cleanup {
5831     my ($storecfg, $statfile) = @_;
5832
5833     print STDERR "starting cleanup\n";
5834
5835     if (my $fd = IO::File->new($statfile, "r")) {
5836         while (defined(my $line = <$fd>)) {
5837             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5838                 my $volid = $2;
5839                 eval {
5840                     if ($volid =~ m|^/|) {
5841                         unlink $volid || die 'unlink failed\n';
5842                     } else {
5843                         PVE::Storage::vdisk_free($storecfg, $volid);
5844                     }
5845                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5846                 };
5847                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5848             } else {
5849                 print STDERR "unable to parse line in statfile - $line";
5850             }
5851         }
5852         $fd->close();
5853     }
5854 }
5855
5856 sub restore_file_archive {
5857     my ($archive, $vmid, $user, $opts) = @_;
5858
5859     return restore_vma_archive($archive, $vmid, $user, $opts)
5860         if $archive eq '-';
5861
5862     my $info = PVE::Storage::archive_info($archive);
5863     my $format = $opts->{format} // $info->{format};
5864     my $comp = $info->{compression};
5865
5866     # try to detect archive format
5867     if ($format eq 'tar') {
5868         return restore_tar_archive($archive, $vmid, $user, $opts);
5869     } else {
5870         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5871     }
5872 }
5873
5874 # hepler to remove disks that will not be used after restore
5875 my $restore_cleanup_oldconf = sub {
5876     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
5877
5878     PVE::QemuConfig->foreach_volume($oldconf, sub {
5879         my ($ds, $drive) = @_;
5880
5881         return if drive_is_cdrom($drive, 1);
5882
5883         my $volid = $drive->{file};
5884         return if !$volid || $volid =~ m|^/|;
5885
5886         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
5887         return if !$path || !$owner || ($owner != $vmid);
5888
5889         # Note: only delete disk we want to restore
5890         # other volumes will become unused
5891         if ($virtdev_hash->{$ds}) {
5892             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
5893             if (my $err = $@) {
5894                 warn $err;
5895             }
5896         }
5897     });
5898
5899     # delete vmstate files, after the restore we have no snapshots anymore
5900     foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5901         my $snap = $oldconf->{snapshots}->{$snapname};
5902         if ($snap->{vmstate}) {
5903             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
5904             if (my $err = $@) {
5905                 warn $err;
5906             }
5907         }
5908     }
5909 };
5910
5911 # Helper to parse vzdump backup device hints
5912 #
5913 # $rpcenv: Environment, used to ckeck storage permissions
5914 # $user: User ID, to check storage permissions
5915 # $storecfg: Storage configuration
5916 # $fh: the file handle for reading the configuration
5917 # $devinfo: should contain device sizes for all backu-up'ed devices
5918 # $options: backup options (pool, default storage)
5919 #
5920 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
5921 my $parse_backup_hints = sub {
5922     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
5923
5924     my $virtdev_hash = {};
5925
5926     while (defined(my $line = <$fh>)) {
5927         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5928             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5929             die "archive does not contain data for drive '$virtdev'\n"
5930                 if !$devinfo->{$devname};
5931
5932             if (defined($options->{storage})) {
5933                 $storeid = $options->{storage} || 'local';
5934             } elsif (!$storeid) {
5935                 $storeid = 'local';
5936             }
5937             $format = 'raw' if !$format;
5938             $devinfo->{$devname}->{devname} = $devname;
5939             $devinfo->{$devname}->{virtdev} = $virtdev;
5940             $devinfo->{$devname}->{format} = $format;
5941             $devinfo->{$devname}->{storeid} = $storeid;
5942
5943             # check permission on storage
5944             my $pool = $options->{pool}; # todo: do we need that?
5945             if ($user ne 'root@pam') {
5946                 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5947             }
5948
5949             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5950         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
5951             my $virtdev = $1;
5952             my $drive = parse_drive($virtdev, $2);
5953             if (drive_is_cloudinit($drive)) {
5954                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5955                 $storeid = $options->{storage} if defined ($options->{storage});
5956                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5957                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
5958
5959                 $virtdev_hash->{$virtdev} = {
5960                     format => $format,
5961                     storeid => $storeid,
5962                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
5963                     is_cloudinit => 1,
5964                 };
5965             }
5966         }
5967     }
5968
5969     return $virtdev_hash;
5970 };
5971
5972 # Helper to allocate and activate all volumes required for a restore
5973 #
5974 # $storecfg: Storage configuration
5975 # $virtdev_hash: as returned by parse_backup_hints()
5976 #
5977 # Returns: { $virtdev => $volid }
5978 my $restore_allocate_devices = sub {
5979     my ($storecfg, $virtdev_hash, $vmid) = @_;
5980
5981     my $map = {};
5982     foreach my $virtdev (sort keys %$virtdev_hash) {
5983         my $d = $virtdev_hash->{$virtdev};
5984         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5985         my $storeid = $d->{storeid};
5986         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5987
5988         # test if requested format is supported
5989         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5990         my $supported = grep { $_ eq $d->{format} } @$validFormats;
5991         $d->{format} = $defFormat if !$supported;
5992
5993         my $name;
5994         if ($d->{is_cloudinit}) {
5995             $name = "vm-$vmid-cloudinit";
5996             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5997             if ($scfg->{path}) {
5998                 $name .= ".$d->{format}";
5999             }
6000         }
6001
6002         my $volid = PVE::Storage::vdisk_alloc(
6003             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6004
6005         print STDERR "new volume ID is '$volid'\n";
6006         $d->{volid} = $volid;
6007
6008         PVE::Storage::activate_volumes($storecfg, [$volid]);
6009
6010         $map->{$virtdev} = $volid;
6011     }
6012
6013     return $map;
6014 };
6015
6016 sub restore_update_config_line {
6017     my ($cookie, $map, $line, $unique) = @_;
6018
6019     return '' if $line =~ m/^\#qmdump\#/;
6020     return '' if $line =~ m/^\#vzdump\#/;
6021     return '' if $line =~ m/^lock:/;
6022     return '' if $line =~ m/^unused\d+:/;
6023     return '' if $line =~ m/^parent:/;
6024
6025     my $res = '';
6026
6027     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6028     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6029         # try to convert old 1.X settings
6030         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6031         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6032             my ($model, $macaddr) = split(/\=/, $devconfig);
6033             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6034             my $net = {
6035                 model => $model,
6036                 bridge => "vmbr$ind",
6037                 macaddr => $macaddr,
6038             };
6039             my $netstr = print_net($net);
6040
6041             $res .= "net$cookie->{netcount}: $netstr\n";
6042             $cookie->{netcount}++;
6043         }
6044     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6045         my ($id, $netstr) = ($1, $2);
6046         my $net = parse_net($netstr);
6047         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6048         $netstr = print_net($net);
6049         $res .= "$id: $netstr\n";
6050     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
6051         my $virtdev = $1;
6052         my $value = $3;
6053         my $di = parse_drive($virtdev, $value);
6054         if (defined($di->{backup}) && !$di->{backup}) {
6055             $res .= "#$line";
6056         } elsif ($map->{$virtdev}) {
6057             delete $di->{format}; # format can change on restore
6058             $di->{file} = $map->{$virtdev};
6059             $value = print_drive($di);
6060             $res .= "$virtdev: $value\n";
6061         } else {
6062             $res .= $line;
6063         }
6064     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6065         my $vmgenid = $1;
6066         if ($vmgenid ne '0') {
6067             # always generate a new vmgenid if there was a valid one setup
6068             $vmgenid = generate_uuid();
6069         }
6070         $res .= "vmgenid: $vmgenid\n";
6071     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6072         my ($uuid, $uuid_str);
6073         UUID::generate($uuid);
6074         UUID::unparse($uuid, $uuid_str);
6075         my $smbios1 = parse_smbios1($2);
6076         $smbios1->{uuid} = $uuid_str;
6077         $res .= $1.print_smbios1($smbios1)."\n";
6078     } else {
6079         $res .= $line;
6080     }
6081
6082     return $res;
6083 }
6084
6085 my $restore_deactivate_volumes = sub {
6086     my ($storecfg, $devinfo) = @_;
6087
6088     my $vollist = [];
6089     foreach my $devname (keys %$devinfo) {
6090         my $volid = $devinfo->{$devname}->{volid};
6091         push @$vollist, $volid if $volid;
6092     }
6093
6094     PVE::Storage::deactivate_volumes($storecfg, $vollist);
6095 };
6096
6097 my $restore_destroy_volumes = sub {
6098     my ($storecfg, $devinfo) = @_;
6099
6100     foreach my $devname (keys %$devinfo) {
6101         my $volid = $devinfo->{$devname}->{volid};
6102         next if !$volid;
6103         eval {
6104             if ($volid =~ m|^/|) {
6105                 unlink $volid || die 'unlink failed\n';
6106             } else {
6107                 PVE::Storage::vdisk_free($storecfg, $volid);
6108             }
6109             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6110         };
6111         print STDERR "unable to cleanup '$volid' - $@" if $@;
6112     }
6113 };
6114
6115 sub scan_volids {
6116     my ($cfg, $vmid) = @_;
6117
6118     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6119
6120     my $volid_hash = {};
6121     foreach my $storeid (keys %$info) {
6122         foreach my $item (@{$info->{$storeid}}) {
6123             next if !($item->{volid} && $item->{size});
6124             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6125             $volid_hash->{$item->{volid}} = $item;
6126         }
6127     }
6128
6129     return $volid_hash;
6130 }
6131
6132 sub update_disk_config {
6133     my ($vmid, $conf, $volid_hash) = @_;
6134
6135     my $changes;
6136     my $prefix = "VM $vmid";
6137
6138     # used and unused disks
6139     my $referenced = {};
6140
6141     # Note: it is allowed to define multiple storages with same path (alias), so
6142     # we need to check both 'volid' and real 'path' (two different volid can point
6143     # to the same path).
6144
6145     my $referencedpath = {};
6146
6147     # update size info
6148     PVE::QemuConfig->foreach_volume($conf, sub {
6149         my ($opt, $drive) = @_;
6150
6151         my $volid = $drive->{file};
6152         return if !$volid;
6153         my $volume = $volid_hash->{$volid};
6154
6155         # mark volid as "in-use" for next step
6156         $referenced->{$volid} = 1;
6157         if ($volume && (my $path = $volume->{path})) {
6158             $referencedpath->{$path} = 1;
6159         }
6160
6161         return if drive_is_cdrom($drive);
6162         return if !$volume;
6163
6164         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6165         if (defined($updated)) {
6166             $changes = 1;
6167             $conf->{$opt} = print_drive($updated);
6168             print "$prefix ($opt): $msg\n";
6169         }
6170     });
6171
6172     # remove 'unusedX' entry if volume is used
6173     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6174         my ($opt, $drive) = @_;
6175
6176         my $volid = $drive->{file};
6177         return if !$volid;
6178
6179         my $path;
6180         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6181         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6182             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6183             $changes = 1;
6184             delete $conf->{$opt};
6185         }
6186
6187         $referenced->{$volid} = 1;
6188         $referencedpath->{$path} = 1 if $path;
6189     });
6190
6191     foreach my $volid (sort keys %$volid_hash) {
6192         next if $volid =~ m/vm-$vmid-state-/;
6193         next if $referenced->{$volid};
6194         my $path = $volid_hash->{$volid}->{path};
6195         next if !$path; # just to be sure
6196         next if $referencedpath->{$path};
6197         $changes = 1;
6198         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6199         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6200         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6201     }
6202
6203     return $changes;
6204 }
6205
6206 sub rescan {
6207     my ($vmid, $nolock, $dryrun) = @_;
6208
6209     my $cfg = PVE::Storage::config();
6210
6211     print "rescan volumes...\n";
6212     my $volid_hash = scan_volids($cfg, $vmid);
6213
6214     my $updatefn =  sub {
6215         my ($vmid) = @_;
6216
6217         my $conf = PVE::QemuConfig->load_config($vmid);
6218
6219         PVE::QemuConfig->check_lock($conf);
6220
6221         my $vm_volids = {};
6222         foreach my $volid (keys %$volid_hash) {
6223             my $info = $volid_hash->{$volid};
6224             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6225         }
6226
6227         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6228
6229         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6230     };
6231
6232     if (defined($vmid)) {
6233         if ($nolock) {
6234             &$updatefn($vmid);
6235         } else {
6236             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6237         }
6238     } else {
6239         my $vmlist = config_list();
6240         foreach my $vmid (keys %$vmlist) {
6241             if ($nolock) {
6242                 &$updatefn($vmid);
6243             } else {
6244                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6245             }
6246         }
6247     }
6248 }
6249
6250 sub restore_proxmox_backup_archive {
6251     my ($archive, $vmid, $user, $options) = @_;
6252
6253     my $storecfg = PVE::Storage::config();
6254
6255     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6256     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6257
6258     my $fingerprint = $scfg->{fingerprint};
6259     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6260
6261     my $repo = PVE::PBSClient::get_repository($scfg);
6262
6263     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6264     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6265     local $ENV{PBS_PASSWORD} = $password;
6266     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6267
6268     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6269         PVE::Storage::parse_volname($storecfg, $archive);
6270
6271     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6272
6273     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6274
6275     my $tmpdir = "/var/tmp/vzdumptmp$$";
6276     rmtree $tmpdir;
6277     mkpath $tmpdir;
6278
6279     my $conffile = PVE::QemuConfig->config_file($vmid);
6280      # disable interrupts (always do cleanups)
6281     local $SIG{INT} =
6282         local $SIG{TERM} =
6283         local $SIG{QUIT} =
6284         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6285
6286     # Note: $oldconf is undef if VM does not exists
6287     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6288     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6289     my $new_conf_raw = '';
6290
6291     my $rpcenv = PVE::RPCEnvironment::get();
6292     my $devinfo = {};
6293
6294     eval {
6295         # enable interrupts
6296         local $SIG{INT} =
6297             local $SIG{TERM} =
6298             local $SIG{QUIT} =
6299             local $SIG{HUP} =
6300             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6301
6302         my $cfgfn = "$tmpdir/qemu-server.conf";
6303         my $firewall_config_fn = "$tmpdir/fw.conf";
6304         my $index_fn = "$tmpdir/index.json";
6305
6306         my $cmd = "restore";
6307
6308         my $param = [$pbs_backup_name, "index.json", $index_fn];
6309         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6310         my $index = PVE::Tools::file_get_contents($index_fn);
6311         $index = decode_json($index);
6312
6313         # print Dumper($index);
6314         foreach my $info (@{$index->{files}}) {
6315             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6316                 my $devname = $1;
6317                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6318                     $devinfo->{$devname}->{size} = $1;
6319                 } else {
6320                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6321                 }
6322             }
6323         }
6324
6325         my $is_qemu_server_backup = scalar(
6326             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6327         );
6328         if (!$is_qemu_server_backup) {
6329             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6330         }
6331         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6332
6333         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6334         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6335
6336         if ($has_firewall_config) {
6337             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6338             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6339
6340             my $pve_firewall_dir = '/etc/pve/firewall';
6341             mkdir $pve_firewall_dir; # make sure the dir exists
6342             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6343         }
6344
6345         my $fh = IO::File->new($cfgfn, "r") ||
6346             die "unable to read qemu-server.conf - $!\n";
6347
6348         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6349
6350         # fixme: rate limit?
6351
6352         # create empty/temp config
6353         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6354
6355         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6356
6357         # allocate volumes
6358         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6359
6360         if (!$options->{live}) {
6361             foreach my $virtdev (sort keys %$virtdev_hash) {
6362                 my $d = $virtdev_hash->{$virtdev};
6363                 next if $d->{is_cloudinit}; # no need to restore cloudinit
6364
6365                 my $volid = $d->{volid};
6366
6367                 my $path = PVE::Storage::path($storecfg, $volid);
6368
6369                 my $pbs_restore_cmd = [
6370                     '/usr/bin/pbs-restore',
6371                     '--repository', $repo,
6372                     $pbs_backup_name,
6373                     "$d->{devname}.img.fidx",
6374                     $path,
6375                     '--verbose',
6376                     ];
6377
6378                 push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6379                 push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
6380
6381                 if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6382                     push @$pbs_restore_cmd, '--skip-zero';
6383                 }
6384
6385                 my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6386                 print "restore proxmox backup image: $dbg_cmdstring\n";
6387                 run_command($pbs_restore_cmd);
6388             }
6389         }
6390
6391         $fh->seek(0, 0) || die "seek failed - $!\n";
6392
6393         my $cookie = { netcount => 0 };
6394         while (defined(my $line = <$fh>)) {
6395             $new_conf_raw .= restore_update_config_line(
6396                 $cookie,
6397                 $map,
6398                 $line,
6399                 $options->{unique},
6400             );
6401         }
6402
6403         $fh->close();
6404     };
6405     my $err = $@;
6406
6407     if ($err || !$options->{live}) {
6408         $restore_deactivate_volumes->($storecfg, $devinfo);
6409     }
6410
6411     rmtree $tmpdir;
6412
6413     if ($err) {
6414         $restore_destroy_volumes->($storecfg, $devinfo);
6415         die $err;
6416     }
6417
6418     if ($options->{live}) {
6419         # keep lock during live-restore
6420         $new_conf_raw .= "\nlock: create";
6421     }
6422
6423     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6424
6425     PVE::Cluster::cfs_update(); # make sure we read new file
6426
6427     eval { rescan($vmid, 1); };
6428     warn $@ if $@;
6429
6430     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
6431
6432     if ($options->{live}) {
6433         # enable interrupts
6434         local $SIG{INT} =
6435             local $SIG{TERM} =
6436             local $SIG{QUIT} =
6437             local $SIG{HUP} =
6438             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
6439
6440         my $conf = PVE::QemuConfig->load_config($vmid);
6441         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
6442
6443         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $repo, $keyfile, $pbs_backup_name);
6444
6445         PVE::QemuConfig->remove_lock($vmid, "create");
6446     }
6447 }
6448
6449 sub pbs_live_restore {
6450     my ($vmid, $conf, $storecfg, $restored_disks, $repo, $keyfile, $snap) = @_;
6451
6452     print "starting VM for live-restore\n";
6453     print "repository: '$repo', snapshot: '$snap'\n";
6454
6455     my $pbs_backing = {};
6456     for my $ds (keys %$restored_disks) {
6457         $ds =~ m/^drive-(.*)$/;
6458         my $confname = $1;
6459         $pbs_backing->{$confname} = {
6460             repository => $repo,
6461             snapshot => $snap,
6462             archive => "$ds.img.fidx",
6463         };
6464         $pbs_backing->{$confname}->{keyfile} = $keyfile if -e $keyfile;
6465
6466         my $drive = parse_drive($confname, $conf->{$confname});
6467         print "restoring '$ds' to '$drive->{file}'\n";
6468     }
6469
6470     my $drives_streamed = 0;
6471     eval {
6472         # make sure HA doesn't interrupt our restore by stopping the VM
6473         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
6474             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
6475         }
6476
6477         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
6478         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
6479         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
6480
6481         my $qmeventd_fd = register_qmeventd_handle($vmid);
6482
6483         # begin streaming, i.e. data copy from PBS to target disk for every vol,
6484         # this will effectively collapse the backing image chain consisting of
6485         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
6486         # removes itself once all backing images vanish with 'auto-remove=on')
6487         my $jobs = {};
6488         for my $ds (sort keys %$restored_disks) {
6489             my $job_id = "restore-$ds";
6490             mon_cmd($vmid, 'block-stream',
6491                 'job-id' => $job_id,
6492                 device => "$ds",
6493             );
6494             $jobs->{$job_id} = {};
6495         }
6496
6497         mon_cmd($vmid, 'cont');
6498         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
6499
6500         print "restore-drive jobs finished successfully, removing all tracking block devices"
6501             ." to disconnect from Proxmox Backup Server\n";
6502
6503         for my $ds (sort keys %$restored_disks) {
6504             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
6505         }
6506
6507         close($qmeventd_fd);
6508     };
6509
6510     my $err = $@;
6511
6512     if ($err) {
6513         warn "An error occured during live-restore: $err\n";
6514         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
6515         die "live-restore failed\n";
6516     }
6517 }
6518
6519 sub restore_vma_archive {
6520     my ($archive, $vmid, $user, $opts, $comp) = @_;
6521
6522     my $readfrom = $archive;
6523
6524     my $cfg = PVE::Storage::config();
6525     my $commands = [];
6526     my $bwlimit = $opts->{bwlimit};
6527
6528     my $dbg_cmdstring = '';
6529     my $add_pipe = sub {
6530         my ($cmd) = @_;
6531         push @$commands, $cmd;
6532         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6533         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6534         $readfrom = '-';
6535     };
6536
6537     my $input = undef;
6538     if ($archive eq '-') {
6539         $input = '<&STDIN';
6540     } else {
6541         # If we use a backup from a PVE defined storage we also consider that
6542         # storage's rate limit:
6543         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6544         if (defined($volid)) {
6545             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6546             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6547             if ($readlimit) {
6548                 print STDERR "applying read rate limit: $readlimit\n";
6549                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6550                 $add_pipe->($cstream);
6551             }
6552         }
6553     }
6554
6555     if ($comp) {
6556         my $info = PVE::Storage::decompressor_info('vma', $comp);
6557         my $cmd = $info->{decompressor};
6558         push @$cmd, $readfrom;
6559         $add_pipe->($cmd);
6560     }
6561
6562     my $tmpdir = "/var/tmp/vzdumptmp$$";
6563     rmtree $tmpdir;
6564
6565     # disable interrupts (always do cleanups)
6566     local $SIG{INT} =
6567         local $SIG{TERM} =
6568         local $SIG{QUIT} =
6569         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6570
6571     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6572     POSIX::mkfifo($mapfifo, 0600);
6573     my $fifofh;
6574     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
6575
6576     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6577
6578     my $oldtimeout;
6579     my $timeout = 5;
6580
6581     my $devinfo = {};
6582
6583     my $rpcenv = PVE::RPCEnvironment::get();
6584
6585     my $conffile = PVE::QemuConfig->config_file($vmid);
6586
6587     # Note: $oldconf is undef if VM does not exist
6588     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6589     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6590     my $new_conf_raw = '';
6591
6592     my %storage_limits;
6593
6594     my $print_devmap = sub {
6595         my $cfgfn = "$tmpdir/qemu-server.conf";
6596
6597         # we can read the config - that is already extracted
6598         my $fh = IO::File->new($cfgfn, "r") ||
6599             die "unable to read qemu-server.conf - $!\n";
6600
6601         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6602         if (-f $fwcfgfn) {
6603             my $pve_firewall_dir = '/etc/pve/firewall';
6604             mkdir $pve_firewall_dir; # make sure the dir exists
6605             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6606         }
6607
6608         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6609
6610         foreach my $info (values %{$virtdev_hash}) {
6611             my $storeid = $info->{storeid};
6612             next if defined($storage_limits{$storeid});
6613
6614             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
6615             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
6616             $storage_limits{$storeid} = $limit * 1024;
6617         }
6618
6619         foreach my $devname (keys %$devinfo) {
6620             die "found no device mapping information for device '$devname'\n"
6621                 if !$devinfo->{$devname}->{virtdev};
6622         }
6623
6624         # create empty/temp config
6625         if ($oldconf) {
6626             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6627             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6628         }
6629
6630         # allocate volumes
6631         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6632
6633         # print restore information to $fifofh
6634         foreach my $virtdev (sort keys %$virtdev_hash) {
6635             my $d = $virtdev_hash->{$virtdev};
6636             next if $d->{is_cloudinit}; # no need to restore cloudinit
6637
6638             my $storeid = $d->{storeid};
6639             my $volid = $d->{volid};
6640
6641             my $map_opts = '';
6642             if (my $limit = $storage_limits{$storeid}) {
6643                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6644             }
6645
6646             my $write_zeros = 1;
6647             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6648                 $write_zeros = 0;
6649             }
6650
6651             my $path = PVE::Storage::path($cfg, $volid);
6652
6653             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6654
6655             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6656         }
6657
6658         $fh->seek(0, 0) || die "seek failed - $!\n";
6659
6660         my $cookie = { netcount => 0 };
6661         while (defined(my $line = <$fh>)) {
6662             $new_conf_raw .= restore_update_config_line(
6663                 $cookie,
6664                 $map,
6665                 $line,
6666                 $opts->{unique},
6667             );
6668         }
6669
6670         $fh->close();
6671     };
6672
6673     eval {
6674         # enable interrupts
6675         local $SIG{INT} =
6676             local $SIG{TERM} =
6677             local $SIG{QUIT} =
6678             local $SIG{HUP} =
6679             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6680         local $SIG{ALRM} = sub { die "got timeout\n"; };
6681
6682         $oldtimeout = alarm($timeout);
6683
6684         my $parser = sub {
6685             my $line = shift;
6686
6687             print "$line\n";
6688
6689             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6690                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6691                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6692             } elsif ($line =~ m/^CTIME: /) {
6693                 # we correctly received the vma config, so we can disable
6694                 # the timeout now for disk allocation (set to 10 minutes, so
6695                 # that we always timeout if something goes wrong)
6696                 alarm(600);
6697                 &$print_devmap();
6698                 print $fifofh "done\n";
6699                 my $tmp = $oldtimeout || 0;
6700                 $oldtimeout = undef;
6701                 alarm($tmp);
6702                 close($fifofh);
6703                 $fifofh = undef;
6704             }
6705         };
6706
6707         print "restore vma archive: $dbg_cmdstring\n";
6708         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6709     };
6710     my $err = $@;
6711
6712     alarm($oldtimeout) if $oldtimeout;
6713
6714     $restore_deactivate_volumes->($cfg, $devinfo);
6715
6716     close($fifofh) if $fifofh;
6717     unlink $mapfifo;
6718     rmtree $tmpdir;
6719
6720     if ($err) {
6721         $restore_destroy_volumes->($cfg, $devinfo);
6722         die $err;
6723     }
6724
6725     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6726
6727     PVE::Cluster::cfs_update(); # make sure we read new file
6728
6729     eval { rescan($vmid, 1); };
6730     warn $@ if $@;
6731
6732     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
6733 }
6734
6735 sub restore_tar_archive {
6736     my ($archive, $vmid, $user, $opts) = @_;
6737
6738     if ($archive ne '-') {
6739         my $firstfile = tar_archive_read_firstfile($archive);
6740         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
6741             if $firstfile ne 'qemu-server.conf';
6742     }
6743
6744     my $storecfg = PVE::Storage::config();
6745
6746     # avoid zombie disks when restoring over an existing VM -> cleanup first
6747     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6748     # skiplock=1 because qmrestore has set the 'create' lock itself already
6749     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6750     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6751
6752     my $tocmd = "/usr/lib/qemu-server/qmextract";
6753
6754     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6755     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6756     $tocmd .= ' --prealloc' if $opts->{prealloc};
6757     $tocmd .= ' --info' if $opts->{info};
6758
6759     # tar option "xf" does not autodetect compression when read from STDIN,
6760     # so we pipe to zcat
6761     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6762         PVE::Tools::shellquote("--to-command=$tocmd");
6763
6764     my $tmpdir = "/var/tmp/vzdumptmp$$";
6765     mkpath $tmpdir;
6766
6767     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6768     local $ENV{VZDUMP_VMID} = $vmid;
6769     local $ENV{VZDUMP_USER} = $user;
6770
6771     my $conffile = PVE::QemuConfig->config_file($vmid);
6772     my $new_conf_raw = '';
6773
6774     # disable interrupts (always do cleanups)
6775     local $SIG{INT} =
6776         local $SIG{TERM} =
6777         local $SIG{QUIT} =
6778         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6779
6780     eval {
6781         # enable interrupts
6782         local $SIG{INT} =
6783             local $SIG{TERM} =
6784             local $SIG{QUIT} =
6785             local $SIG{HUP} =
6786             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6787
6788         if ($archive eq '-') {
6789             print "extracting archive from STDIN\n";
6790             run_command($cmd, input => "<&STDIN");
6791         } else {
6792             print "extracting archive '$archive'\n";
6793             run_command($cmd);
6794         }
6795
6796         return if $opts->{info};
6797
6798         # read new mapping
6799         my $map = {};
6800         my $statfile = "$tmpdir/qmrestore.stat";
6801         if (my $fd = IO::File->new($statfile, "r")) {
6802             while (defined (my $line = <$fd>)) {
6803                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6804                     $map->{$1} = $2 if $1;
6805                 } else {
6806                     print STDERR "unable to parse line in statfile - $line\n";
6807                 }
6808             }
6809             $fd->close();
6810         }
6811
6812         my $confsrc = "$tmpdir/qemu-server.conf";
6813
6814         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
6815
6816         my $cookie = { netcount => 0 };
6817         while (defined (my $line = <$srcfd>)) {
6818             $new_conf_raw .= restore_update_config_line(
6819                 $cookie,
6820                 $map,
6821                 $line,
6822                 $opts->{unique},
6823             );
6824         }
6825
6826         $srcfd->close();
6827     };
6828     if (my $err = $@) {
6829         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6830         die $err;
6831     }
6832
6833     rmtree $tmpdir;
6834
6835     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6836
6837     PVE::Cluster::cfs_update(); # make sure we read new file
6838
6839     eval { rescan($vmid, 1); };
6840     warn $@ if $@;
6841 };
6842
6843 sub foreach_storage_used_by_vm {
6844     my ($conf, $func) = @_;
6845
6846     my $sidhash = {};
6847
6848     PVE::QemuConfig->foreach_volume($conf, sub {
6849         my ($ds, $drive) = @_;
6850         return if drive_is_cdrom($drive);
6851
6852         my $volid = $drive->{file};
6853
6854         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6855         $sidhash->{$sid} = $sid if $sid;
6856     });
6857
6858     foreach my $sid (sort keys %$sidhash) {
6859         &$func($sid);
6860     }
6861 }
6862
6863 my $qemu_snap_storage = {
6864     rbd => 1,
6865 };
6866 sub do_snapshots_with_qemu {
6867     my ($storecfg, $volid) = @_;
6868
6869     my $storage_name = PVE::Storage::parse_volume_id($volid);
6870     my $scfg = $storecfg->{ids}->{$storage_name};
6871     die "could not find storage '$storage_name'\n" if !defined($scfg);
6872
6873     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6874         return 1;
6875     }
6876
6877     if ($volid =~ m/\.(qcow2|qed)$/){
6878         return 1;
6879     }
6880
6881     return;
6882 }
6883
6884 sub qga_check_running {
6885     my ($vmid, $nowarn) = @_;
6886
6887     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6888     if ($@) {
6889         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6890         return 0;
6891     }
6892     return 1;
6893 }
6894
6895 sub template_create {
6896     my ($vmid, $conf, $disk) = @_;
6897
6898     my $storecfg = PVE::Storage::config();
6899
6900     PVE::QemuConfig->foreach_volume($conf, sub {
6901         my ($ds, $drive) = @_;
6902
6903         return if drive_is_cdrom($drive);
6904         return if $disk && $ds ne $disk;
6905
6906         my $volid = $drive->{file};
6907         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6908
6909         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6910         $drive->{file} = $voliddst;
6911         $conf->{$ds} = print_drive($drive);
6912         PVE::QemuConfig->write_config($vmid, $conf);
6913     });
6914 }
6915
6916 sub convert_iscsi_path {
6917     my ($path) = @_;
6918
6919     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6920         my $portal = $1;
6921         my $target = $2;
6922         my $lun = $3;
6923
6924         my $initiator_name = get_initiator_name();
6925
6926         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6927                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6928     }
6929
6930     die "cannot convert iscsi path '$path', unkown format\n";
6931 }
6932
6933 sub qemu_img_convert {
6934     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6935
6936     my $storecfg = PVE::Storage::config();
6937     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6938     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6939
6940     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6941
6942     my $cachemode;
6943     my $src_path;
6944     my $src_is_iscsi = 0;
6945     my $src_format;
6946
6947     if ($src_storeid) {
6948         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6949         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6950         $src_format = qemu_img_format($src_scfg, $src_volname);
6951         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6952         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6953         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6954     } elsif (-f $src_volid) {
6955         $src_path = $src_volid;
6956         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6957             $src_format = $1;
6958         }
6959     }
6960
6961     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6962
6963     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6964     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6965     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6966     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6967
6968     my $cmd = [];
6969     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6970     push @$cmd, '-l', "snapshot.name=$snapname"
6971         if $snapname && $src_format && $src_format eq "qcow2";
6972     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6973     push @$cmd, '-T', $cachemode if defined($cachemode);
6974
6975     if ($src_is_iscsi) {
6976         push @$cmd, '--image-opts';
6977         $src_path = convert_iscsi_path($src_path);
6978     } elsif ($src_format) {
6979         push @$cmd, '-f', $src_format;
6980     }
6981
6982     if ($dst_is_iscsi) {
6983         push @$cmd, '--target-image-opts';
6984         $dst_path = convert_iscsi_path($dst_path);
6985     } else {
6986         push @$cmd, '-O', $dst_format;
6987     }
6988
6989     push @$cmd, $src_path;
6990
6991     if (!$dst_is_iscsi && $is_zero_initialized) {
6992         push @$cmd, "zeroinit:$dst_path";
6993     } else {
6994         push @$cmd, $dst_path;
6995     }
6996
6997     my $parser = sub {
6998         my $line = shift;
6999         if($line =~ m/\((\S+)\/100\%\)/){
7000             my $percent = $1;
7001             my $transferred = int($size * $percent / 100);
7002             my $total_h = render_bytes($size, 1);
7003             my $transferred_h = render_bytes($transferred, 1);
7004
7005             print "transferred $transferred_h of $total_h ($percent%)\n";
7006         }
7007
7008     };
7009
7010     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7011     my $err = $@;
7012     die "copy failed: $err" if $err;
7013 }
7014
7015 sub qemu_img_format {
7016     my ($scfg, $volname) = @_;
7017
7018     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7019         return $1;
7020     } else {
7021         return "raw";
7022     }
7023 }
7024
7025 sub qemu_drive_mirror {
7026     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7027
7028     $jobs = {} if !$jobs;
7029
7030     my $qemu_target;
7031     my $format;
7032     $jobs->{"drive-$drive"} = {};
7033
7034     if ($dst_volid =~ /^nbd:/) {
7035         $qemu_target = $dst_volid;
7036         $format = "nbd";
7037     } else {
7038         my $storecfg = PVE::Storage::config();
7039         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7040
7041         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7042
7043         $format = qemu_img_format($dst_scfg, $dst_volname);
7044
7045         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7046
7047         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7048     }
7049
7050     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7051     $opts->{format} = $format if $format;
7052
7053     if (defined($src_bitmap)) {
7054         $opts->{sync} = 'incremental';
7055         $opts->{bitmap} = $src_bitmap;
7056         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7057     }
7058
7059     if (defined($bwlimit)) {
7060         $opts->{speed} = $bwlimit * 1024;
7061         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7062     } else {
7063         print "drive mirror is starting for drive-$drive\n";
7064     }
7065
7066     # if a job already runs for this device we get an error, catch it for cleanup
7067     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7068     if (my $err = $@) {
7069         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7070         warn "$@\n" if $@;
7071         die "mirroring error: $err\n";
7072     }
7073
7074     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7075 }
7076
7077 # $completion can be either
7078 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7079 # 'cancel': wait until all jobs are ready, block-job-cancel them
7080 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7081 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7082 sub qemu_drive_mirror_monitor {
7083     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7084
7085     $completion //= 'complete';
7086     $op //= "mirror";
7087
7088     eval {
7089         my $err_complete = 0;
7090
7091         my $starttime = time ();
7092         while (1) {
7093             die "block job ('$op') timed out\n" if $err_complete > 300;
7094
7095             my $stats = mon_cmd($vmid, "query-block-jobs");
7096             my $ctime = time();
7097
7098             my $running_jobs = {};
7099             for my $stat (@$stats) {
7100                 next if $stat->{type} ne $op;
7101                 $running_jobs->{$stat->{device}} = $stat;
7102             }
7103
7104             my $readycounter = 0;
7105
7106             for my $job_id (sort keys %$jobs) {
7107                 my $job = $running_jobs->{$job_id};
7108
7109                 my $vanished = !defined($job);
7110                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7111                 if($complete || ($vanished && $completion eq 'auto')) {
7112                     print "$job_id: $op-job finished\n";
7113                     delete $jobs->{$job_id};
7114                     next;
7115                 }
7116
7117                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7118
7119                 my $busy = $job->{busy};
7120                 my $ready = $job->{ready};
7121                 if (my $total = $job->{len}) {
7122                     my $transferred = $job->{offset} || 0;
7123                     my $remaining = $total - $transferred;
7124                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7125
7126                     my $duration = $ctime - $starttime;
7127                     my $total_h = render_bytes($total, 1);
7128                     my $transferred_h = render_bytes($transferred, 1);
7129
7130                     my $status = sprintf(
7131                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7132                         render_duration($duration),
7133                     );
7134
7135                     if ($ready) {
7136                         if ($busy) {
7137                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7138                         } else {
7139                             $status .= ", ready";
7140                         }
7141                     }
7142                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7143                     $jobs->{$job_id}->{ready} = $ready;
7144                 }
7145
7146                 $readycounter++ if $job->{ready};
7147             }
7148
7149             last if scalar(keys %$jobs) == 0;
7150
7151             if ($readycounter == scalar(keys %$jobs)) {
7152                 print "all '$op' jobs are ready\n";
7153
7154                 # do the complete later (or has already been done)
7155                 last if $completion eq 'skip' || $completion eq 'auto';
7156
7157                 if ($vmiddst && $vmiddst != $vmid) {
7158                     my $agent_running = $qga && qga_check_running($vmid);
7159                     if ($agent_running) {
7160                         print "freeze filesystem\n";
7161                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7162                     } else {
7163                         print "suspend vm\n";
7164                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7165                     }
7166
7167                     # if we clone a disk for a new target vm, we don't switch the disk
7168                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7169
7170                     if ($agent_running) {
7171                         print "unfreeze filesystem\n";
7172                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7173                     } else {
7174                         print "resume vm\n";
7175                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
7176                     }
7177
7178                     last;
7179                 } else {
7180
7181                     for my $job_id (sort keys %$jobs) {
7182                         # try to switch the disk if source and destination are on the same guest
7183                         print "$job_id: Completing block job_id...\n";
7184
7185                         my $op;
7186                         if ($completion eq 'complete') {
7187                             $op = 'block-job-complete';
7188                         } elsif ($completion eq 'cancel') {
7189                             $op = 'block-job-cancel';
7190                         } else {
7191                             die "invalid completion value: $completion\n";
7192                         }
7193                         eval { mon_cmd($vmid, $op, device => $job_id) };
7194                         if ($@ =~ m/cannot be completed/) {
7195                             print "$job_id: block job cannot be completed, trying again.\n";
7196                             $err_complete++;
7197                         }else {
7198                             print "$job_id: Completed successfully.\n";
7199                             $jobs->{$job_id}->{complete} = 1;
7200                         }
7201                     }
7202                 }
7203             }
7204             sleep 1;
7205         }
7206     };
7207     my $err = $@;
7208
7209     if ($err) {
7210         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7211         die "block job ($op) error: $err";
7212     }
7213 }
7214
7215 sub qemu_blockjobs_cancel {
7216     my ($vmid, $jobs) = @_;
7217
7218     foreach my $job (keys %$jobs) {
7219         print "$job: Cancelling block job\n";
7220         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7221         $jobs->{$job}->{cancel} = 1;
7222     }
7223
7224     while (1) {
7225         my $stats = mon_cmd($vmid, "query-block-jobs");
7226
7227         my $running_jobs = {};
7228         foreach my $stat (@$stats) {
7229             $running_jobs->{$stat->{device}} = $stat;
7230         }
7231
7232         foreach my $job (keys %$jobs) {
7233
7234             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7235                 print "$job: Done.\n";
7236                 delete $jobs->{$job};
7237             }
7238         }
7239
7240         last if scalar(keys %$jobs) == 0;
7241
7242         sleep 1;
7243     }
7244 }
7245
7246 sub clone_disk {
7247     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7248         $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
7249
7250     my $newvolid;
7251
7252     if (!$full) {
7253         print "create linked clone of drive $drivename ($drive->{file})\n";
7254         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7255         push @$newvollist, $newvolid;
7256     } else {
7257
7258         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7259         $storeid = $storage if $storage;
7260
7261         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7262
7263         print "create full clone of drive $drivename ($drive->{file})\n";
7264         my $name = undef;
7265         my $size = undef;
7266         if (drive_is_cloudinit($drive)) {
7267             $name = "vm-$newvmid-cloudinit";
7268             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7269             if ($scfg->{path}) {
7270                 $name .= ".$dst_format";
7271             }
7272             $snapname = undef;
7273             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7274         } elsif ($drivename eq 'efidisk0') {
7275             $size = get_efivars_size($conf);
7276         } else {
7277             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
7278         }
7279         $newvolid = PVE::Storage::vdisk_alloc(
7280             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
7281         );
7282         push @$newvollist, $newvolid;
7283
7284         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7285
7286         if (drive_is_cloudinit($drive)) {
7287             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
7288             # if this is the case, we have to complete any block-jobs still there from
7289             # previous drive-mirrors
7290             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
7291                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
7292             }
7293             goto no_data_clone;
7294         }
7295
7296         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7297         if (!$running || $snapname) {
7298             # TODO: handle bwlimits
7299             if ($drivename eq 'efidisk0') {
7300                 # the relevant data on the efidisk may be smaller than the source
7301                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
7302                 # that is given by the OVMF_VARS.fd
7303                 my $src_path = PVE::Storage::path($storecfg, $drive->{file});
7304                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
7305
7306                 # better for Ceph if block size is not too small, see bug #3324
7307                 my $bs = 1024*1024;
7308
7309                 run_command(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=$bs", "osize=$size",
7310                     "if=$src_path", "of=$dst_path"]);
7311             } else {
7312                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7313             }
7314         } else {
7315
7316             my $kvmver = get_running_qemu_version ($vmid);
7317             if (!min_version($kvmver, 2, 7)) {
7318                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7319                     if $drive->{iothread};
7320             }
7321
7322             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7323                 $completion, $qga, $bwlimit);
7324         }
7325     }
7326
7327 no_data_clone:
7328     my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
7329
7330     my $disk = $drive;
7331     $disk->{format} = undef;
7332     $disk->{file} = $newvolid;
7333     $disk->{size} = $size if defined($size);
7334
7335     return $disk;
7336 }
7337
7338 sub get_running_qemu_version {
7339     my ($vmid) = @_;
7340     my $res = mon_cmd($vmid, "query-version");
7341     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7342 }
7343
7344 sub qemu_use_old_bios_files {
7345     my ($machine_type) = @_;
7346
7347     return if !$machine_type;
7348
7349     my $use_old_bios_files = undef;
7350
7351     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7352         $machine_type = $1;
7353         $use_old_bios_files = 1;
7354     } else {
7355         my $version = extract_version($machine_type, kvm_user_version());
7356         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7357         # load new efi bios files on migration. So this hack is required to allow
7358         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7359         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7360         $use_old_bios_files = !min_version($version, 2, 4);
7361     }
7362
7363     return ($use_old_bios_files, $machine_type);
7364 }
7365
7366 sub get_efivars_size {
7367     my ($conf) = @_;
7368     my $arch = get_vm_arch($conf);
7369     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7370     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
7371     return -s $ovmf_vars;
7372 }
7373
7374 sub update_efidisk_size {
7375     my ($conf) = @_;
7376
7377     return if !defined($conf->{efidisk0});
7378
7379     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
7380     $disk->{size} = get_efivars_size($conf);
7381     $conf->{efidisk0} = print_drive($disk);
7382
7383     return;
7384 }
7385
7386 sub create_efidisk($$$$$) {
7387     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7388
7389     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7390     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7391
7392     my $vars_size_b = -s $ovmf_vars;
7393     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7394     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7395     PVE::Storage::activate_volumes($storecfg, [$volid]);
7396
7397     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7398     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7399
7400     return ($volid, $size/1024);
7401 }
7402
7403 sub vm_iothreads_list {
7404     my ($vmid) = @_;
7405
7406     my $res = mon_cmd($vmid, 'query-iothreads');
7407
7408     my $iothreads = {};
7409     foreach my $iothread (@$res) {
7410         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7411     }
7412
7413     return $iothreads;
7414 }
7415
7416 sub scsihw_infos {
7417     my ($conf, $drive) = @_;
7418
7419     my $maxdev = 0;
7420
7421     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7422         $maxdev = 7;
7423     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7424         $maxdev = 1;
7425     } else {
7426         $maxdev = 256;
7427     }
7428
7429     my $controller = int($drive->{index} / $maxdev);
7430     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
7431         ? "virtioscsi"
7432         : "scsihw";
7433
7434     return ($maxdev, $controller, $controller_prefix);
7435 }
7436
7437 sub windows_version {
7438     my ($ostype) = @_;
7439
7440     return 0 if !$ostype;
7441
7442     my $winversion = 0;
7443
7444     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7445         $winversion = 5;
7446     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7447         $winversion = 6;
7448     } elsif ($ostype =~ m/^win(\d+)$/) {
7449         $winversion = $1;
7450     }
7451
7452     return $winversion;
7453 }
7454
7455 sub resolve_dst_disk_format {
7456         my ($storecfg, $storeid, $src_volname, $format) = @_;
7457         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7458
7459         if (!$format) {
7460             # if no target format is specified, use the source disk format as hint
7461             if ($src_volname) {
7462                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7463                 $format = qemu_img_format($scfg, $src_volname);
7464             } else {
7465                 return $defFormat;
7466             }
7467         }
7468
7469         # test if requested format is supported - else use default
7470         my $supported = grep { $_ eq $format } @$validFormats;
7471         $format = $defFormat if !$supported;
7472         return $format;
7473 }
7474
7475 # NOTE: if this logic changes, please update docs & possibly gui logic
7476 sub find_vmstate_storage {
7477     my ($conf, $storecfg) = @_;
7478
7479     # first, return storage from conf if set
7480     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7481
7482     my ($target, $shared, $local);
7483
7484     foreach_storage_used_by_vm($conf, sub {
7485         my ($sid) = @_;
7486         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7487         my $dst = $scfg->{shared} ? \$shared : \$local;
7488         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7489     });
7490
7491     # second, use shared storage where VM has at least one disk
7492     # third, use local storage where VM has at least one disk
7493     # fall back to local storage
7494     $target = $shared // $local // 'local';
7495
7496     return $target;
7497 }
7498
7499 sub generate_uuid {
7500     my ($uuid, $uuid_str);
7501     UUID::generate($uuid);
7502     UUID::unparse($uuid, $uuid_str);
7503     return $uuid_str;
7504 }
7505
7506 sub generate_smbios1_uuid {
7507     return "uuid=".generate_uuid();
7508 }
7509
7510 sub nbd_stop {
7511     my ($vmid) = @_;
7512
7513     mon_cmd($vmid, 'nbd-server-stop');
7514 }
7515
7516 sub create_reboot_request {
7517     my ($vmid) = @_;
7518     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7519         or die "failed to create reboot trigger file: $!\n";
7520     close($fh);
7521 }
7522
7523 sub clear_reboot_request {
7524     my ($vmid) = @_;
7525     my $path = "/run/qemu-server/$vmid.reboot";
7526     my $res = 0;
7527
7528     $res = unlink($path);
7529     die "could not remove reboot request for $vmid: $!"
7530         if !$res && $! != POSIX::ENOENT;
7531
7532     return $res;
7533 }
7534
7535 sub bootorder_from_legacy {
7536     my ($conf, $bootcfg) = @_;
7537
7538     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
7539     my $bootindex_hash = {};
7540     my $i = 1;
7541     foreach my $o (split(//, $boot)) {
7542         $bootindex_hash->{$o} = $i*100;
7543         $i++;
7544     }
7545
7546     my $bootorder = {};
7547
7548     PVE::QemuConfig->foreach_volume($conf, sub {
7549         my ($ds, $drive) = @_;
7550
7551         if (drive_is_cdrom ($drive, 1)) {
7552             if ($bootindex_hash->{d}) {
7553                 $bootorder->{$ds} = $bootindex_hash->{d};
7554                 $bootindex_hash->{d} += 1;
7555             }
7556         } elsif ($bootindex_hash->{c}) {
7557             $bootorder->{$ds} = $bootindex_hash->{c}
7558                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
7559             $bootindex_hash->{c} += 1;
7560         }
7561     });
7562
7563     if ($bootindex_hash->{n}) {
7564         for (my $i = 0; $i < $MAX_NETS; $i++) {
7565             my $netname = "net$i";
7566             next if !$conf->{$netname};
7567             $bootorder->{$netname} = $bootindex_hash->{n};
7568             $bootindex_hash->{n} += 1;
7569         }
7570     }
7571
7572     return $bootorder;
7573 }
7574
7575 # Generate default device list for 'boot: order=' property. Matches legacy
7576 # default boot order, but with explicit device names. This is important, since
7577 # the fallback for when neither 'order' nor the old format is specified relies
7578 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
7579 sub get_default_bootdevices {
7580     my ($conf) = @_;
7581
7582     my @ret = ();
7583
7584     # harddisk
7585     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
7586     push @ret, $first if $first;
7587
7588     # cdrom
7589     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
7590     push @ret, $first if $first;
7591
7592     # network
7593     for (my $i = 0; $i < $MAX_NETS; $i++) {
7594         my $netname = "net$i";
7595         next if !$conf->{$netname};
7596         push @ret, $netname;
7597         last;
7598     }
7599
7600     return \@ret;
7601 }
7602
7603 sub device_bootorder {
7604     my ($conf) = @_;
7605
7606     return bootorder_from_legacy($conf) if !defined($conf->{boot});
7607
7608     my $boot = parse_property_string($boot_fmt, $conf->{boot});
7609
7610     my $bootorder = {};
7611     if (!defined($boot) || $boot->{legacy}) {
7612         $bootorder = bootorder_from_legacy($conf, $boot);
7613     } elsif ($boot->{order}) {
7614         my $i = 100; # start at 100 to allow user to insert devices before us with -args
7615         for my $dev (PVE::Tools::split_list($boot->{order})) {
7616             $bootorder->{$dev} = $i++;
7617         }
7618     }
7619
7620     return $bootorder;
7621 }
7622
7623 sub register_qmeventd_handle {
7624     my ($vmid) = @_;
7625
7626     my $fh;
7627     my $peer = "/var/run/qmeventd.sock";
7628     my $count = 0;
7629
7630     for (;;) {
7631         $count++;
7632         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
7633         last if $fh;
7634         if ($! != EINTR && $! != EAGAIN) {
7635             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
7636         }
7637         if ($count > 4) {
7638             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
7639               . "after $count retries\n";
7640         }
7641         usleep(25000);
7642     }
7643
7644     # send handshake to mark VM as backing up
7645     print $fh to_json({vzdump => {vmid => "$vmid"}});
7646
7647     # return handle to be closed later when inhibit is no longer required
7648     return $fh;
7649 }
7650
7651 # bash completion helper
7652
7653 sub complete_backup_archives {
7654     my ($cmdname, $pname, $cvalue) = @_;
7655
7656     my $cfg = PVE::Storage::config();
7657
7658     my $storeid;
7659
7660     if ($cvalue =~ m/^([^:]+):/) {
7661         $storeid = $1;
7662     }
7663
7664     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7665
7666     my $res = [];
7667     foreach my $id (keys %$data) {
7668         foreach my $item (@{$data->{$id}}) {
7669             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
7670             push @$res, $item->{volid} if defined($item->{volid});
7671         }
7672     }
7673
7674     return $res;
7675 }
7676
7677 my $complete_vmid_full = sub {
7678     my ($running) = @_;
7679
7680     my $idlist = vmstatus();
7681
7682     my $res = [];
7683
7684     foreach my $id (keys %$idlist) {
7685         my $d = $idlist->{$id};
7686         if (defined($running)) {
7687             next if $d->{template};
7688             next if $running && $d->{status} ne 'running';
7689             next if !$running && $d->{status} eq 'running';
7690         }
7691         push @$res, $id;
7692
7693     }
7694     return $res;
7695 };
7696
7697 sub complete_vmid {
7698     return &$complete_vmid_full();
7699 }
7700
7701 sub complete_vmid_stopped {
7702     return &$complete_vmid_full(0);
7703 }
7704
7705 sub complete_vmid_running {
7706     return &$complete_vmid_full(1);
7707 }
7708
7709 sub complete_storage {
7710
7711     my $cfg = PVE::Storage::config();
7712     my $ids = $cfg->{ids};
7713
7714     my $res = [];
7715     foreach my $sid (keys %$ids) {
7716         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7717         next if !$ids->{$sid}->{content}->{images};
7718         push @$res, $sid;
7719     }
7720
7721     return $res;
7722 }
7723
7724 sub complete_migration_storage {
7725     my ($cmd, $param, $current_value, $all_args) = @_;
7726
7727     my $targetnode = @$all_args[1];
7728
7729     my $cfg = PVE::Storage::config();
7730     my $ids = $cfg->{ids};
7731
7732     my $res = [];
7733     foreach my $sid (keys %$ids) {
7734         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7735         next if !$ids->{$sid}->{content}->{images};
7736         push @$res, $sid;
7737     }
7738
7739     return $res;
7740 }
7741
7742 sub vm_is_paused {
7743     my ($vmid) = @_;
7744     my $qmpstatus = eval {
7745         PVE::QemuConfig::assert_config_exists_on_node($vmid);
7746         mon_cmd($vmid, "query-status");
7747     };
7748     warn "$@\n" if $@;
7749     return $qmpstatus && $qmpstatus->{status} eq "paused";
7750 }
7751
7752 sub check_volume_storage_type {
7753     my ($storecfg, $vol) = @_;
7754
7755     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
7756     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7757     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
7758
7759     die "storage '$storeid' does not support content-type '$vtype'\n"
7760         if !$scfg->{content}->{$vtype};
7761
7762     return 1;
7763 }
7764
7765 1;