PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::CGroup;
  31 use PVE::DataCenterConfig;
  32 use PVE::Exception qw(raise raise_param_exc);
  33 use PVE::Format qw(render_duration render_bytes);
  34 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  35 use PVE::INotify;
  36 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  37 use PVE::ProcFSTools;
  38 use PVE::PBSClient;
  39 use PVE::RPCEnvironment;
  40 use PVE::Storage;
  41 use PVE::SysFSTools;
  42 use PVE::Systemd;
  43 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  44
  45 use PVE::QMPClient;
  46 use PVE::QemuConfig;
  47 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  48 use PVE::QemuServer::Cloudinit;
  49 use PVE::QemuServer::CGroup;
  50 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  51 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  52 use PVE::QemuServer::Machine;
  53 use PVE::QemuServer::Memory;
  54 use PVE::QemuServer::Monitor qw(mon_cmd);
  55 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  56 use PVE::QemuServer::USB qw(parse_usb_device);
  57
  58 my $have_sdn;
  59 eval {
  60     require PVE::Network::SDN::Zones;
  61     $have_sdn = 1;
  62 };
  63
  64 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  65 my $OVMF = {
  66     x86_64 => [
  67         "$EDK2_FW_BASE/OVMF_CODE.fd",
  68         "$EDK2_FW_BASE/OVMF_VARS.fd"
  69     ],
  70     aarch64 => [
  71         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  72         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  73     ],
  74 };
  75
  76 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  77
  78 # Note about locking: we use flock on the config file protect
  79 # against concurent actions.
  80 # Aditionaly, we have a 'lock' setting in the config file. This
  81 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  82 # allowed when such lock is set. But you can ignore this kind of
  83 # lock with the --skiplock flag.
  84
  85 cfs_register_file('/qemu-server/',
  86                   \&parse_vm_config,
  87                   \&write_vm_config);
  88
  89 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  90     description => "Some command save/restore state from this location.",
  91     type => 'string',
  92     maxLength => 128,
  93     optional => 1,
  94 });
  95
  96 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  97         description => "Specifies the Qemu machine type.",
  98         type => 'string',
  99         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 100         maxLength => 40,
 101         optional => 1,
 102 });
 103
 104
 105 sub map_storage {
 106     my ($map, $source) = @_;
 107
 108     return $source if !defined($map);
 109
 110     return $map->{entries}->{$source}
 111         if $map->{entries} && defined($map->{entries}->{$source});
 112
 113     return $map->{default} if $map->{default};
 114
 115     # identity (fallback)
 116     return $source;
 117 }
 118
 119 PVE::JSONSchema::register_standard_option('pve-targetstorage', {
 120     description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
 121     type => 'string',
 122     format => 'storagepair-list',
 123     optional => 1,
 124 });
 125
 126 #no warnings 'redefine';
 127
 128 my $nodename_cache;
 129 sub nodename {
 130     $nodename_cache //= PVE::INotify::nodename();
 131     return $nodename_cache;
 132 }
 133
 134 my $watchdog_fmt = {
 135     model => {
 136         default_key => 1,
 137         type => 'string',
 138         enum => [qw(i6300esb ib700)],
 139         description => "Watchdog type to emulate.",
 140         default => 'i6300esb',
 141         optional => 1,
 142     },
 143     action => {
 144         type => 'string',
 145         enum => [qw(reset shutdown poweroff pause debug none)],
 146         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 147         optional => 1,
 148     },
 149 };
 150 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 151
 152 my $agent_fmt = {
 153     enabled => {
 154         description => "Enable/disable Qemu GuestAgent.",
 155         type => 'boolean',
 156         default => 0,
 157         default_key => 1,
 158     },
 159     fstrim_cloned_disks => {
 160         description => "Run fstrim after moving a disk or migrating the VM.",
 161         type => 'boolean',
 162         optional => 1,
 163         default => 0
 164     },
 165     type => {
 166         description => "Select the agent type",
 167         type => 'string',
 168         default => 'virtio',
 169         optional => 1,
 170         enum => [qw(virtio isa)],
 171     },
 172 };
 173
 174 my $vga_fmt = {
 175     type => {
 176         description => "Select the VGA type.",
 177         type => 'string',
 178         default => 'std',
 179         optional => 1,
 180         default_key => 1,
 181         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 182     },
 183     memory => {
 184         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 185         type => 'integer',
 186         optional => 1,
 187         minimum => 4,
 188         maximum => 512,
 189     },
 190 };
 191
 192 my $ivshmem_fmt = {
 193     size => {
 194         type => 'integer',
 195         minimum => 1,
 196         description => "The size of the file in MB.",
 197     },
 198     name => {
 199         type => 'string',
 200         pattern => '[a-zA-Z0-9\-]+',
 201         optional => 1,
 202         format_description => 'string',
 203         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 204     },
 205 };
 206
 207 my $audio_fmt = {
 208     device => {
 209         type => 'string',
 210         enum => [qw(ich9-intel-hda intel-hda AC97)],
 211         description =>  "Configure an audio device."
 212     },
 213     driver =>  {
 214         type => 'string',
 215         enum => ['spice', 'none'],
 216         default => 'spice',
 217         optional => 1,
 218         description => "Driver backend for the audio device."
 219     },
 220 };
 221
 222 my $spice_enhancements_fmt = {
 223     foldersharing => {
 224         type => 'boolean',
 225         optional => 1,
 226         default => '0',
 227         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 228     },
 229     videostreaming =>  {
 230         type => 'string',
 231         enum => ['off', 'all', 'filter'],
 232         default => 'off',
 233         optional => 1,
 234         description => "Enable video streaming. Uses compression for detected video streams."
 235     },
 236 };
 237
 238 my $rng_fmt = {
 239     source => {
 240         type => 'string',
 241         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 242         default_key => 1,
 243         description => "The file on the host to gather entropy from. In most"
 244                      . " cases /dev/urandom should be preferred over /dev/random"
 245                      . " to avoid entropy-starvation issues on the host. Using"
 246                      . " urandom does *not* decrease security in any meaningful"
 247                      . " way, as it's still seeded from real entropy, and the"
 248                      . " bytes provided will most likely be mixed with real"
 249                      . " entropy on the guest as well. /dev/hwrng can be used"
 250                      . " to pass through a hardware RNG from the host.",
 251     },
 252     max_bytes => {
 253         type => 'integer',
 254         description => "Maximum bytes of entropy injected into the guest every"
 255                      . " 'period' milliseconds. Prefer a lower value when using"
 256                      . " /dev/random as source. Use 0 to disable limiting"
 257                      . " (potentially dangerous!).",
 258         optional => 1,
 259
 260         # default is 1 KiB/s, provides enough entropy to the guest to avoid
 261         # boot-starvation issues (e.g. systemd etc...) while allowing no chance
 262         # of overwhelming the host, provided we're reading from /dev/urandom
 263         default => 1024,
 264     },
 265     period => {
 266         type => 'integer',
 267         description => "Every 'period' milliseconds the entropy-injection quota"
 268                      . " is reset, allowing the guest to retrieve another"
 269                      . " 'max_bytes' of entropy.",
 270         optional => 1,
 271         default => 1000,
 272     },
 273 };
 274
 275 my $confdesc = {
 276     onboot => {
 277         optional => 1,
 278         type => 'boolean',
 279         description => "Specifies whether a VM will be started during system bootup.",
 280         default => 0,
 281     },
 282     autostart => {
 283         optional => 1,
 284         type => 'boolean',
 285         description => "Automatic restart after crash (currently ignored).",
 286         default => 0,
 287     },
 288     hotplug => {
 289         optional => 1,
 290         type => 'string', format => 'pve-hotplug-features',
 291         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 292         default => 'network,disk,usb',
 293     },
 294     reboot => {
 295         optional => 1,
 296         type => 'boolean',
 297         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 298         default => 1,
 299     },
 300     lock => {
 301         optional => 1,
 302         type => 'string',
 303         description => "Lock/unlock the VM.",
 304         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 305     },
 306     cpulimit => {
 307         optional => 1,
 308         type => 'number',
 309         description => "Limit of CPU usage.",
 310         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 311         minimum => 0,
 312         maximum => 128,
 313         default => 0,
 314     },
 315     cpuunits => {
 316         optional => 1,
 317         type => 'integer',
 318         description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
 319         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
 320             ." The larger the number is, the more CPU time this VM gets. Number is relative to"
 321             ." weights of all the other running VMs.",
 322         minimum => 2,
 323         maximum => 262144,
 324         default => 'cgroup v1: 1024, cgroup v2: 100',
 325     },
 326     memory => {
 327         optional => 1,
 328         type => 'integer',
 329         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 330         minimum => 16,
 331         default => 512,
 332     },
 333     balloon => {
 334         optional => 1,
 335         type => 'integer',
 336         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 337         minimum => 0,
 338     },
 339     shares => {
 340         optional => 1,
 341         type => 'integer',
 342         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 343         minimum => 0,
 344         maximum => 50000,
 345         default => 1000,
 346     },
 347     keyboard => {
 348         optional => 1,
 349         type => 'string',
 350         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 351                        "It should not be necessary to set it.",
 352         enum => PVE::Tools::kvmkeymaplist(),
 353         default => undef,
 354     },
 355     name => {
 356         optional => 1,
 357         type => 'string', format => 'dns-name',
 358         description => "Set a name for the VM. Only used on the configuration web interface.",
 359     },
 360     scsihw => {
 361         optional => 1,
 362         type => 'string',
 363         description => "SCSI controller model",
 364         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 365         default => 'lsi',
 366     },
 367     description => {
 368         optional => 1,
 369         type => 'string',
 370         description => "Description for the VM. Shown in the web-interface VM's summary."
 371             ." This is saved as comment inside the configuration file.",
 372         maxLength => 1024 * 8,
 373     },
 374     ostype => {
 375         optional => 1,
 376         type => 'string',
 377         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 378         description => "Specify guest operating system.",
 379         verbose_description => <<EODESC,
 380 Specify guest operating system. This is used to enable special
 381 optimization/features for specific operating systems:
 382
 383 [horizontal]
 384 other;; unspecified OS
 385 wxp;; Microsoft Windows XP
 386 w2k;; Microsoft Windows 2000
 387 w2k3;; Microsoft Windows 2003
 388 w2k8;; Microsoft Windows 2008
 389 wvista;; Microsoft Windows Vista
 390 win7;; Microsoft Windows 7
 391 win8;; Microsoft Windows 8/2012/2012r2
 392 win10;; Microsoft Windows 10/2016/2019
 393 l24;; Linux 2.4 Kernel
 394 l26;; Linux 2.6 - 5.X Kernel
 395 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 396 EODESC
 397     },
 398     boot => {
 399         optional => 1,
 400         type => 'string', format => 'pve-qm-boot',
 401         description => "Specify guest boot order. Use with 'order=', usage with"
 402                      . " no key or 'legacy=' is deprecated.",
 403     },
 404     bootdisk => {
 405         optional => 1,
 406         type => 'string', format => 'pve-qm-bootdisk',
 407         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 408         pattern => '(ide|sata|scsi|virtio)\d+',
 409     },
 410     smp => {
 411         optional => 1,
 412         type => 'integer',
 413         description => "The number of CPUs. Please use option -sockets instead.",
 414         minimum => 1,
 415         default => 1,
 416     },
 417     sockets => {
 418         optional => 1,
 419         type => 'integer',
 420         description => "The number of CPU sockets.",
 421         minimum => 1,
 422         default => 1,
 423     },
 424     cores => {
 425         optional => 1,
 426         type => 'integer',
 427         description => "The number of cores per socket.",
 428         minimum => 1,
 429         default => 1,
 430     },
 431     numa => {
 432         optional => 1,
 433         type => 'boolean',
 434         description => "Enable/disable NUMA.",
 435         default => 0,
 436     },
 437     hugepages => {
 438         optional => 1,
 439         type => 'string',
 440         description => "Enable/disable hugepages memory.",
 441         enum => [qw(any 2 1024)],
 442     },
 443     keephugepages => {
 444         optional => 1,
 445         type => 'boolean',
 446         default => 0,
 447         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 448             ." after VM shutdown and can be used for subsequent starts.",
 449     },
 450     vcpus => {
 451         optional => 1,
 452         type => 'integer',
 453         description => "Number of hotplugged vcpus.",
 454         minimum => 1,
 455         default => 0,
 456     },
 457     acpi => {
 458         optional => 1,
 459         type => 'boolean',
 460         description => "Enable/disable ACPI.",
 461         default => 1,
 462     },
 463     agent => {
 464         optional => 1,
 465         description => "Enable/disable Qemu GuestAgent and its properties.",
 466         type => 'string',
 467         format => $agent_fmt,
 468     },
 469     kvm => {
 470         optional => 1,
 471         type => 'boolean',
 472         description => "Enable/disable KVM hardware virtualization.",
 473         default => 1,
 474     },
 475     tdf => {
 476         optional => 1,
 477         type => 'boolean',
 478         description => "Enable/disable time drift fix.",
 479         default => 0,
 480     },
 481     localtime => {
 482         optional => 1,
 483         type => 'boolean',
 484         description => "Set the real time clock to local time. This is enabled by default if ostype"
 485             ." indicates a Microsoft OS.",
 486     },
 487     freeze => {
 488         optional => 1,
 489         type => 'boolean',
 490         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 491     },
 492     vga => {
 493         optional => 1,
 494         type => 'string', format => $vga_fmt,
 495         description => "Configure the VGA hardware.",
 496         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 497             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 498             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 499             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 500             ." display server. For win* OS you can select how many independent displays you want,"
 501             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 502             ." using a serial device as terminal.",
 503     },
 504     watchdog => {
 505         optional => 1,
 506         type => 'string', format => 'pve-qm-watchdog',
 507         description => "Create a virtual hardware watchdog device.",
 508         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 509             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 510             ." else the watchdog will reset the guest (or execute the respective action specified)",
 511     },
 512     startdate => {
 513         optional => 1,
 514         type => 'string',
 515         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 516         description => "Set the initial date of the real time clock. Valid format for date are:"
 517             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 518         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 519         default => 'now',
 520     },
 521     startup =>  get_standard_option('pve-startup-order'),
 522     template => {
 523         optional => 1,
 524         type => 'boolean',
 525         description => "Enable/disable Template.",
 526         default => 0,
 527     },
 528     args => {
 529         optional => 1,
 530         type => 'string',
 531         description => "Arbitrary arguments passed to kvm.",
 532         verbose_description => <<EODESCR,
 533 Arbitrary arguments passed to kvm, for example:
 534
 535 args: -no-reboot -no-hpet
 536
 537 NOTE: this option is for experts only.
 538 EODESCR
 539     },
 540     tablet => {
 541         optional => 1,
 542         type => 'boolean',
 543         default => 1,
 544         description => "Enable/disable the USB tablet device.",
 545         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 546             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 547             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 548             ." may consider disabling this to save some context switches. This is turned off by"
 549             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 550     },
 551     migrate_speed => {
 552         optional => 1,
 553         type => 'integer',
 554         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 555         minimum => 0,
 556         default => 0,
 557     },
 558     migrate_downtime => {
 559         optional => 1,
 560         type => 'number',
 561         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 562         minimum => 0,
 563         default => 0.1,
 564     },
 565     cdrom => {
 566         optional => 1,
 567         type => 'string', format => 'pve-qm-ide',
 568         typetext => '<volume>',
 569         description => "This is an alias for option -ide2",
 570     },
 571     cpu => {
 572         optional => 1,
 573         description => "Emulated CPU type.",
 574         type => 'string',
 575         format => 'pve-vm-cpu-conf',
 576     },
 577     parent => get_standard_option('pve-snapshot-name', {
 578         optional => 1,
 579         description => "Parent snapshot name. This is used internally, and should not be modified.",
 580     }),
 581     snaptime => {
 582         optional => 1,
 583         description => "Timestamp for snapshots.",
 584         type => 'integer',
 585         minimum => 0,
 586     },
 587     vmstate => {
 588         optional => 1,
 589         type => 'string', format => 'pve-volume-id',
 590         description => "Reference to a volume which stores the VM state. This is used internally"
 591             ." for snapshots.",
 592     },
 593     vmstatestorage => get_standard_option('pve-storage-id', {
 594         description => "Default storage for VM state volumes/files.",
 595         optional => 1,
 596     }),
 597     runningmachine => get_standard_option('pve-qemu-machine', {
 598         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 599             ." for snapshots.",
 600     }),
 601     runningcpu => {
 602         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 603             ." internally for snapshots.",
 604         optional => 1,
 605         type => 'string',
 606         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 607         format_description => 'QEMU -cpu parameter'
 608     },
 609     machine => get_standard_option('pve-qemu-machine'),
 610     arch => {
 611         description => "Virtual processor architecture. Defaults to the host.",
 612         optional => 1,
 613         type => 'string',
 614         enum => [qw(x86_64 aarch64)],
 615     },
 616     smbios1 => {
 617         description => "Specify SMBIOS type 1 fields.",
 618         type => 'string', format => 'pve-qm-smbios1',
 619         maxLength => 512,
 620         optional => 1,
 621     },
 622     protection => {
 623         optional => 1,
 624         type => 'boolean',
 625         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 626             ." remove disk operations.",
 627         default => 0,
 628     },
 629     bios => {
 630         optional => 1,
 631         type => 'string',
 632         enum => [ qw(seabios ovmf) ],
 633         description => "Select BIOS implementation.",
 634         default => 'seabios',
 635     },
 636     vmgenid => {
 637         type => 'string',
 638         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 639         format_description => 'UUID',
 640         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 641             ." to disable explicitly.",
 642         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 643             ." value identifier to the guest OS. This allows to notify the guest operating system"
 644             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 645             ." execution or creation from a template). The guest operating system notices the"
 646             ." change, and is then able to react as appropriate by marking its copies of"
 647             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 648             ."Note that auto-creation only works when done through API/CLI create or update methods"
 649             .", but not when manually editing the config file.",
 650         default => "1 (autogenerated)",
 651         optional => 1,
 652     },
 653     hookscript => {
 654         type => 'string',
 655         format => 'pve-volume-id',
 656         optional => 1,
 657         description => "Script that will be executed during various steps in the vms lifetime.",
 658     },
 659     ivshmem => {
 660         type => 'string',
 661         format => $ivshmem_fmt,
 662         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 663             ." the host.",
 664         optional => 1,
 665     },
 666     audio0 => {
 667         type => 'string',
 668         format => $audio_fmt,
 669         description => "Configure a audio device, useful in combination with QXL/Spice.",
 670         optional => 1
 671     },
 672     spice_enhancements => {
 673         type => 'string',
 674         format => $spice_enhancements_fmt,
 675         description => "Configure additional enhancements for SPICE.",
 676         optional => 1
 677     },
 678     tags => {
 679         type => 'string', format => 'pve-tag-list',
 680         description => 'Tags of the VM. This is only meta information.',
 681         optional => 1,
 682     },
 683     rng0 => {
 684         type => 'string',
 685         format => $rng_fmt,
 686         description => "Configure a VirtIO-based Random Number Generator.",
 687         optional => 1,
 688     },
 689 };
 690
 691 my $cicustom_fmt = {
 692     meta => {
 693         type => 'string',
 694         optional => 1,
 695         description => 'Specify a custom file containing all meta data passed to the VM via"
 696             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 697         format => 'pve-volume-id',
 698         format_description => 'volume',
 699     },
 700     network => {
 701         type => 'string',
 702         optional => 1,
 703         description => 'Specify a custom file containing all network data passed to the VM via'
 704             .' cloud-init.',
 705         format => 'pve-volume-id',
 706         format_description => 'volume',
 707     },
 708     user => {
 709         type => 'string',
 710         optional => 1,
 711         description => 'Specify a custom file containing all user data passed to the VM via'
 712             .' cloud-init.',
 713         format => 'pve-volume-id',
 714         format_description => 'volume',
 715     },
 716 };
 717 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 718
 719 my $confdesc_cloudinit = {
 720     citype => {
 721         optional => 1,
 722         type => 'string',
 723         description => 'Specifies the cloud-init configuration format. The default depends on the'
 724             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 725             .' and `configdrive2` for windows.',
 726         enum => ['configdrive2', 'nocloud', 'opennebula'],
 727     },
 728     ciuser => {
 729         optional => 1,
 730         type => 'string',
 731         description => "cloud-init: User name to change ssh keys and password for instead of the"
 732             ." image's configured default user.",
 733     },
 734     cipassword => {
 735         optional => 1,
 736         type => 'string',
 737         description => 'cloud-init: Password to assign the user. Using this is generally not'
 738             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 739             .' support hashed passwords.',
 740     },
 741     cicustom => {
 742         optional => 1,
 743         type => 'string',
 744         description => 'cloud-init: Specify custom files to replace the automatically generated'
 745             .' ones at start.',
 746         format => 'pve-qm-cicustom',
 747     },
 748     searchdomain => {
 749         optional => 1,
 750         type => 'string',
 751         description => "cloud-init: Sets DNS search domains for a container. Create will'
 752             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 753             .' are set.",
 754     },
 755     nameserver => {
 756         optional => 1,
 757         type => 'string', format => 'address-list',
 758         description => "cloud-init: Sets DNS server IP address for a container. Create will'
 759             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 760             .' are set.",
 761     },
 762     sshkeys => {
 763         optional => 1,
 764         type => 'string',
 765         format => 'urlencoded',
 766         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 767     },
 768 };
 769
 770 # what about other qemu settings ?
 771 #cpu => 'string',
 772 #machine => 'string',
 773 #fda => 'file',
 774 #fdb => 'file',
 775 #mtdblock => 'file',
 776 #sd => 'file',
 777 #pflash => 'file',
 778 #snapshot => 'bool',
 779 #bootp => 'file',
 780 ##tftp => 'dir',
 781 ##smb => 'dir',
 782 #kernel => 'file',
 783 #append => 'string',
 784 #initrd => 'file',
 785 ##soundhw => 'string',
 786
 787 while (my ($k, $v) = each %$confdesc) {
 788     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 789 }
 790
 791 my $MAX_USB_DEVICES = 5;
 792 my $MAX_NETS = 32;
 793 my $MAX_SERIAL_PORTS = 4;
 794 my $MAX_PARALLEL_PORTS = 3;
 795 my $MAX_NUMA = 8;
 796
 797 my $numa_fmt = {
 798     cpus => {
 799         type => "string",
 800         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 801         description => "CPUs accessing this NUMA node.",
 802         format_description => "id[-id];...",
 803     },
 804     memory => {
 805         type => "number",
 806         description => "Amount of memory this NUMA node provides.",
 807         optional => 1,
 808     },
 809     hostnodes => {
 810         type => "string",
 811         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 812         description => "Host NUMA nodes to use.",
 813         format_description => "id[-id];...",
 814         optional => 1,
 815     },
 816     policy => {
 817         type => 'string',
 818         enum => [qw(preferred bind interleave)],
 819         description => "NUMA allocation policy.",
 820         optional => 1,
 821     },
 822 };
 823 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 824 my $numadesc = {
 825     optional => 1,
 826     type => 'string', format => $numa_fmt,
 827     description => "NUMA topology.",
 828 };
 829 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 830
 831 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 832     $confdesc->{"numa$i"} = $numadesc;
 833 }
 834
 835 my $nic_model_list = [
 836     'e1000',
 837     'e1000-82540em',
 838     'e1000-82544gc',
 839     'e1000-82545em',
 840     'e1000e',
 841     'i82551',
 842     'i82557b',
 843     'i82559er',
 844     'ne2k_isa',
 845     'ne2k_pci',
 846     'pcnet',
 847     'rtl8139',
 848     'virtio',
 849     'vmxnet3',
 850 ];
 851 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 852
 853 my $net_fmt_bridge_descr = <<__EOD__;
 854 Bridge to attach the network device to. The Proxmox VE standard bridge
 855 is called 'vmbr0'.
 856
 857 If you do not specify a bridge, we create a kvm user (NATed) network
 858 device, which provides DHCP and DNS services. The following addresses
 859 are used:
 860
 861  10.0.2.2   Gateway
 862  10.0.2.3   DNS Server
 863  10.0.2.4   SMB Server
 864
 865 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 866 __EOD__
 867
 868 my $net_fmt = {
 869     macaddr  => get_standard_option('mac-addr', {
 870         description => "MAC address. That address must be unique withing your network. This is"
 871             ." automatically generated if not specified.",
 872     }),
 873     model => {
 874         type => 'string',
 875         description => "Network Card Model. The 'virtio' model provides the best performance with"
 876             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 877             ." best to use 'e1000'.",
 878         enum => $nic_model_list,
 879         default_key => 1,
 880     },
 881     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 882     bridge => {
 883         type => 'string',
 884         description => $net_fmt_bridge_descr,
 885         format_description => 'bridge',
 886         pattern => '[-_.\w\d]+',
 887         optional => 1,
 888     },
 889     queues => {
 890         type => 'integer',
 891         minimum => 0, maximum => 16,
 892         description => 'Number of packet queues to be used on the device.',
 893         optional => 1,
 894     },
 895     rate => {
 896         type => 'number',
 897         minimum => 0,
 898         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 899         optional => 1,
 900     },
 901     tag => {
 902         type => 'integer',
 903         minimum => 1, maximum => 4094,
 904         description => 'VLAN tag to apply to packets on this interface.',
 905         optional => 1,
 906     },
 907     trunks => {
 908         type => 'string',
 909         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 910         description => 'VLAN trunks to pass through this interface.',
 911         format_description => 'vlanid[;vlanid...]',
 912         optional => 1,
 913     },
 914     firewall => {
 915         type => 'boolean',
 916         description => 'Whether this interface should be protected by the firewall.',
 917         optional => 1,
 918     },
 919     link_down => {
 920         type => 'boolean',
 921         description => 'Whether this interface should be disconnected (like pulling the plug).',
 922         optional => 1,
 923     },
 924     mtu => {
 925         type => 'integer',
 926         minimum => 1, maximum => 65520,
 927         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 928         optional => 1,
 929     },
 930 };
 931
 932 my $netdesc = {
 933     optional => 1,
 934     type => 'string', format => $net_fmt,
 935     description => "Specify network devices.",
 936 };
 937
 938 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 939
 940 my $ipconfig_fmt = {
 941     ip => {
 942         type => 'string',
 943         format => 'pve-ipv4-config',
 944         format_description => 'IPv4Format/CIDR',
 945         description => 'IPv4 address in CIDR format.',
 946         optional => 1,
 947         default => 'dhcp',
 948     },
 949     gw => {
 950         type => 'string',
 951         format => 'ipv4',
 952         format_description => 'GatewayIPv4',
 953         description => 'Default gateway for IPv4 traffic.',
 954         optional => 1,
 955         requires => 'ip',
 956     },
 957     ip6 => {
 958         type => 'string',
 959         format => 'pve-ipv6-config',
 960         format_description => 'IPv6Format/CIDR',
 961         description => 'IPv6 address in CIDR format.',
 962         optional => 1,
 963         default => 'dhcp',
 964     },
 965     gw6 => {
 966         type => 'string',
 967         format => 'ipv6',
 968         format_description => 'GatewayIPv6',
 969         description => 'Default gateway for IPv6 traffic.',
 970         optional => 1,
 971         requires => 'ip6',
 972     },
 973 };
 974 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 975 my $ipconfigdesc = {
 976     optional => 1,
 977     type => 'string', format => 'pve-qm-ipconfig',
 978     description => <<'EODESCR',
 979 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 980
 981 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 982
 983 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
 984 gateway should be provided.
 985 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
 986 cloud-init 19.4 or newer.
 987
 988 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
 989 dhcp on IPv4.
 990 EODESCR
 991 };
 992 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 993
 994 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 995     $confdesc->{"net$i"} = $netdesc;
 996     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 997 }
 998
 999 foreach my $key (keys %$confdesc_cloudinit) {
1000     $confdesc->{$key} = $confdesc_cloudinit->{$key};
1001 }
1002
1003 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1004 sub verify_volume_id_or_qm_path {
1005     my ($volid, $noerr) = @_;
1006
1007     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
1008         return $volid;
1009     }
1010
1011     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
1012     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1013     if ($@) {
1014         return if $noerr;
1015         die $@;
1016     }
1017     return $volid;
1018 }
1019
1020 my $usb_fmt = {
1021     host => {
1022         default_key => 1,
1023         type => 'string', format => 'pve-qm-usb-device',
1024         format_description => 'HOSTUSBDEVICE|spice',
1025         description => <<EODESCR,
1026 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1027
1028  'bus-port(.port)*' (decimal numbers) or
1029  'vendor_id:product_id' (hexadeciaml numbers) or
1030  'spice'
1031
1032 You can use the 'lsusb -t' command to list existing usb devices.
1033
1034 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1035 machines - use with special care.
1036
1037 The value 'spice' can be used to add a usb redirection devices for spice.
1038 EODESCR
1039     },
1040     usb3 => {
1041         optional => 1,
1042         type => 'boolean',
1043         description => "Specifies whether if given host option is a USB3 device or port.",
1044         default => 0,
1045     },
1046 };
1047
1048 my $usbdesc = {
1049     optional => 1,
1050     type => 'string', format => $usb_fmt,
1051     description => "Configure an USB device (n is 0 to 4).",
1052 };
1053 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1054
1055 my $serialdesc = {
1056         optional => 1,
1057         type => 'string',
1058         pattern => '(/dev/.+|socket)',
1059         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1060         verbose_description =>  <<EODESCR,
1061 Create a serial device inside the VM (n is 0 to 3), and pass through a
1062 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1063 host side (use 'qm terminal' to open a terminal connection).
1064
1065 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1066 use with special care.
1067
1068 CAUTION: Experimental! User reported problems with this option.
1069 EODESCR
1070 };
1071
1072 my $paralleldesc= {
1073         optional => 1,
1074         type => 'string',
1075         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1076         description =>  "Map host parallel devices (n is 0 to 2).",
1077         verbose_description =>  <<EODESCR,
1078 Map host parallel devices (n is 0 to 2).
1079
1080 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1081 machines - use with special care.
1082
1083 CAUTION: Experimental! User reported problems with this option.
1084 EODESCR
1085 };
1086
1087 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1088     $confdesc->{"parallel$i"} = $paralleldesc;
1089 }
1090
1091 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1092     $confdesc->{"serial$i"} = $serialdesc;
1093 }
1094
1095 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1096     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1097 }
1098
1099 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1100     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1101 }
1102
1103 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1104     $confdesc->{"usb$i"} = $usbdesc;
1105 }
1106
1107 my $boot_fmt = {
1108     legacy => {
1109         optional => 1,
1110         default_key => 1,
1111         type => 'string',
1112         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1113                      . " Deprecated, use 'order=' instead.",
1114         pattern => '[acdn]{1,4}',
1115         format_description => "[acdn]{1,4}",
1116
1117         # note: this is also the fallback if boot: is not given at all
1118         default => 'cdn',
1119     },
1120     order => {
1121         optional => 1,
1122         type => 'string',
1123         format => 'pve-qm-bootdev-list',
1124         format_description => "device[;device...]",
1125         description => <<EODESC,
1126 The guest will attempt to boot from devices in the order they appear here.
1127
1128 Disks, optical drives and passed-through storage USB devices will be directly
1129 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1130 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1131
1132 Note that only devices in this list will be marked as bootable and thus loaded
1133 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1134 (e.g. software-raid), you need to specify all of them here.
1135
1136 Overrides the deprecated 'legacy=[acdn]*' value when given.
1137 EODESC
1138     },
1139 };
1140 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1141
1142 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1143 sub verify_bootdev {
1144     my ($dev, $noerr) = @_;
1145
1146     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && $dev !~ m/^efidisk/;
1147
1148     my $check = sub {
1149         my ($base) = @_;
1150         return 0 if $dev !~ m/^$base\d+$/;
1151         return 0 if !$confdesc->{$dev};
1152         return 1;
1153     };
1154
1155     return $dev if $check->("net");
1156     return $dev if $check->("usb");
1157     return $dev if $check->("hostpci");
1158
1159     return if $noerr;
1160     die "invalid boot device '$dev'\n";
1161 }
1162
1163 sub print_bootorder {
1164     my ($devs) = @_;
1165     return "" if !@$devs;
1166     my $data = { order => join(';', @$devs) };
1167     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1168 }
1169
1170 my $kvm_api_version = 0;
1171
1172 sub kvm_version {
1173     return $kvm_api_version if $kvm_api_version;
1174
1175     open my $fh, '<', '/dev/kvm' or return;
1176
1177     # 0xae00 => KVM_GET_API_VERSION
1178     $kvm_api_version = ioctl($fh, 0xae00, 0);
1179     close($fh);
1180
1181     return $kvm_api_version;
1182 }
1183
1184 my $kvm_user_version = {};
1185 my $kvm_mtime = {};
1186
1187 sub kvm_user_version {
1188     my ($binary) = @_;
1189
1190     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1191     my $st = stat($binary);
1192
1193     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1194     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1195         $cachedmtime == $st->mtime;
1196
1197     $kvm_user_version->{$binary} = 'unknown';
1198     $kvm_mtime->{$binary} = $st->mtime;
1199
1200     my $code = sub {
1201         my $line = shift;
1202         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1203             $kvm_user_version->{$binary} = $2;
1204         }
1205     };
1206
1207     eval { run_command([$binary, '--version'], outfunc => $code); };
1208     warn $@ if $@;
1209
1210     return $kvm_user_version->{$binary};
1211
1212 }
1213 my sub extract_version {
1214     my ($machine_type, $version) = @_;
1215     $version = kvm_user_version() if !defined($version);
1216     PVE::QemuServer::Machine::extract_version($machine_type, $version)
1217 }
1218
1219 sub kernel_has_vhost_net {
1220     return -c '/dev/vhost-net';
1221 }
1222
1223 sub option_exists {
1224     my $key = shift;
1225     return defined($confdesc->{$key});
1226 }
1227
1228 my $cdrom_path;
1229 sub get_cdrom_path {
1230
1231     return  $cdrom_path if $cdrom_path;
1232
1233     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1234     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1235     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1236 }
1237
1238 sub get_iso_path {
1239     my ($storecfg, $vmid, $cdrom) = @_;
1240
1241     if ($cdrom eq 'cdrom') {
1242         return get_cdrom_path();
1243     } elsif ($cdrom eq 'none') {
1244         return '';
1245     } elsif ($cdrom =~ m|^/|) {
1246         return $cdrom;
1247     } else {
1248         return PVE::Storage::path($storecfg, $cdrom);
1249     }
1250 }
1251
1252 # try to convert old style file names to volume IDs
1253 sub filename_to_volume_id {
1254     my ($vmid, $file, $media) = @_;
1255
1256      if (!($file eq 'none' || $file eq 'cdrom' ||
1257           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1258
1259         return if $file =~ m|/|;
1260
1261         if ($media && $media eq 'cdrom') {
1262             $file = "local:iso/$file";
1263         } else {
1264             $file = "local:$vmid/$file";
1265         }
1266     }
1267
1268     return $file;
1269 }
1270
1271 sub verify_media_type {
1272     my ($opt, $vtype, $media) = @_;
1273
1274     return if !$media;
1275
1276     my $etype;
1277     if ($media eq 'disk') {
1278         $etype = 'images';
1279     } elsif ($media eq 'cdrom') {
1280         $etype = 'iso';
1281     } else {
1282         die "internal error";
1283     }
1284
1285     return if ($vtype eq $etype);
1286
1287     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1288 }
1289
1290 sub cleanup_drive_path {
1291     my ($opt, $storecfg, $drive) = @_;
1292
1293     # try to convert filesystem paths to volume IDs
1294
1295     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1296         ($drive->{file} !~ m|^/dev/.+|) &&
1297         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1298         ($drive->{file} !~ m/^\d+$/)) {
1299         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1300         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1301             if !$vtype;
1302         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1303         verify_media_type($opt, $vtype, $drive->{media});
1304         $drive->{file} = $volid;
1305     }
1306
1307     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1308 }
1309
1310 sub parse_hotplug_features {
1311     my ($data) = @_;
1312
1313     my $res = {};
1314
1315     return $res if $data eq '0';
1316
1317     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1318
1319     foreach my $feature (PVE::Tools::split_list($data)) {
1320         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1321             $res->{$1} = 1;
1322         } else {
1323             die "invalid hotplug feature '$feature'\n";
1324         }
1325     }
1326     return $res;
1327 }
1328
1329 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1330 sub pve_verify_hotplug_features {
1331     my ($value, $noerr) = @_;
1332
1333     return $value if parse_hotplug_features($value);
1334
1335     return if $noerr;
1336
1337     die "unable to parse hotplug option\n";
1338 }
1339
1340 sub scsi_inquiry {
1341     my($fh, $noerr) = @_;
1342
1343     my $SG_IO = 0x2285;
1344     my $SG_GET_VERSION_NUM = 0x2282;
1345
1346     my $versionbuf = "\x00" x 8;
1347     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1348     if (!$ret) {
1349         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1350         return;
1351     }
1352     my $version = unpack("I", $versionbuf);
1353     if ($version < 30000) {
1354         die "scsi generic interface too old\n"  if !$noerr;
1355         return;
1356     }
1357
1358     my $buf = "\x00" x 36;
1359     my $sensebuf = "\x00" x 8;
1360     my $cmd = pack("C x3 C x1", 0x12, 36);
1361
1362     # see /usr/include/scsi/sg.h
1363     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1364
1365     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1366                       length($sensebuf), 0, length($buf), $buf,
1367                       $cmd, $sensebuf, 6000);
1368
1369     $ret = ioctl($fh, $SG_IO, $packet);
1370     if (!$ret) {
1371         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1372         return;
1373     }
1374
1375     my @res = unpack($sg_io_hdr_t, $packet);
1376     if ($res[17] || $res[18]) {
1377         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1378         return;
1379     }
1380
1381     my $res = {};
1382     (my $byte0, my $byte1, $res->{vendor},
1383      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1384
1385     $res->{removable} = $byte1 & 128 ? 1 : 0;
1386     $res->{type} = $byte0 & 31;
1387
1388     return $res;
1389 }
1390
1391 sub path_is_scsi {
1392     my ($path) = @_;
1393
1394     my $fh = IO::File->new("+<$path") || return;
1395     my $res = scsi_inquiry($fh, 1);
1396     close($fh);
1397
1398     return $res;
1399 }
1400
1401 sub print_tabletdevice_full {
1402     my ($conf, $arch) = @_;
1403
1404     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1405
1406     # we use uhci for old VMs because tablet driver was buggy in older qemu
1407     my $usbbus;
1408     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1409         $usbbus = 'ehci';
1410     } else {
1411         $usbbus = 'uhci';
1412     }
1413
1414     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1415 }
1416
1417 sub print_keyboarddevice_full {
1418     my ($conf, $arch, $machine) = @_;
1419
1420     return if $arch ne 'aarch64';
1421
1422     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1423 }
1424
1425 my sub get_drive_id {
1426     my ($drive) = @_;
1427     return "$drive->{interface}$drive->{index}";
1428 }
1429
1430 sub print_drivedevice_full {
1431     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1432
1433     my $device = '';
1434     my $maxdev = 0;
1435
1436     my $drive_id = get_drive_id($drive);
1437     if ($drive->{interface} eq 'virtio') {
1438         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1439         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1440         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1441     } elsif ($drive->{interface} eq 'scsi') {
1442
1443         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1444         my $unit = $drive->{index} % $maxdev;
1445         my $devicetype = 'hd';
1446         my $path = '';
1447         if (drive_is_cdrom($drive)) {
1448             $devicetype = 'cd';
1449         } else {
1450             if ($drive->{file} =~ m|^/|) {
1451                 $path = $drive->{file};
1452                 if (my $info = path_is_scsi($path)) {
1453                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1454                         $devicetype = 'block';
1455                     } elsif ($info->{type} == 1) { # tape
1456                         $devicetype = 'generic';
1457                     }
1458                 }
1459             } else {
1460                  $path = PVE::Storage::path($storecfg, $drive->{file});
1461             }
1462
1463             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1464             my $version = extract_version($machine_type, kvm_user_version());
1465             if ($path =~ m/^iscsi\:\/\// &&
1466                !min_version($version, 4, 1)) {
1467                 $devicetype = 'generic';
1468             }
1469         }
1470
1471         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1472             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1473         } else {
1474             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1475                 .",lun=$drive->{index}";
1476         }
1477         $device .= ",drive=drive-$drive_id,id=$drive_id";
1478
1479         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1480             $device .= ",rotation_rate=1";
1481         }
1482         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1483
1484     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1485         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1486         my $controller = int($drive->{index} / $maxdev);
1487         my $unit = $drive->{index} % $maxdev;
1488         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1489
1490         $device = "ide-$devicetype";
1491         if ($drive->{interface} eq 'ide') {
1492             $device .= ",bus=ide.$controller,unit=$unit";
1493         } else {
1494             $device .= ",bus=ahci$controller.$unit";
1495         }
1496         $device .= ",drive=drive-$drive_id,id=$drive_id";
1497
1498         if ($devicetype eq 'hd') {
1499             if (my $model = $drive->{model}) {
1500                 $model = URI::Escape::uri_unescape($model);
1501                 $device .= ",model=$model";
1502             }
1503             if ($drive->{ssd}) {
1504                 $device .= ",rotation_rate=1";
1505             }
1506         }
1507         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1508     } elsif ($drive->{interface} eq 'usb') {
1509         die "implement me";
1510         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1511     } else {
1512         die "unsupported interface type";
1513     }
1514
1515     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1516
1517     if (my $serial = $drive->{serial}) {
1518         $serial = URI::Escape::uri_unescape($serial);
1519         $device .= ",serial=$serial";
1520     }
1521
1522
1523     return $device;
1524 }
1525
1526 sub get_initiator_name {
1527     my $initiator;
1528
1529     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1530     while (defined(my $line = <$fh>)) {
1531         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1532         $initiator = $1;
1533         last;
1534     }
1535     $fh->close();
1536
1537     return $initiator;
1538 }
1539
1540 sub print_drive_commandline_full {
1541     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1542
1543     my $path;
1544     my $volid = $drive->{file};
1545     my $format = $drive->{format};
1546     my $drive_id = get_drive_id($drive);
1547
1548     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1549     my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1550
1551     if (drive_is_cdrom($drive)) {
1552         $path = get_iso_path($storecfg, $vmid, $volid);
1553         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1554     } else {
1555         if ($storeid) {
1556             $path = PVE::Storage::path($storecfg, $volid);
1557             $format //= qemu_img_format($scfg, $volname);
1558         } else {
1559             $path = $volid;
1560             $format //= "raw";
1561         }
1562    }
1563
1564    my $is_rbd = $path =~ m/^rbd:/;
1565
1566     my $opts = '';
1567     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1568     foreach my $o (@qemu_drive_options) {
1569         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1570     }
1571
1572     # snapshot only accepts on|off
1573     if (defined($drive->{snapshot})) {
1574         my $v = $drive->{snapshot} ? 'on' : 'off';
1575         $opts .= ",snapshot=$v";
1576     }
1577
1578     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1579         my ($dir, $qmpname) = @$type;
1580         if (my $v = $drive->{"mbps$dir"}) {
1581             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1582         }
1583         if (my $v = $drive->{"mbps${dir}_max"}) {
1584             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1585         }
1586         if (my $v = $drive->{"bps${dir}_max_length"}) {
1587             $opts .= ",throttling.bps$qmpname-max-length=$v";
1588         }
1589         if (my $v = $drive->{"iops${dir}"}) {
1590             $opts .= ",throttling.iops$qmpname=$v";
1591         }
1592         if (my $v = $drive->{"iops${dir}_max"}) {
1593             $opts .= ",throttling.iops$qmpname-max=$v";
1594         }
1595         if (my $v = $drive->{"iops${dir}_max_length"}) {
1596             $opts .= ",throttling.iops$qmpname-max-length=$v";
1597         }
1598     }
1599
1600     if ($pbs_name) {
1601         $format = "rbd" if $is_rbd;
1602         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1603             if !$format;
1604         $opts .= ",format=alloc-track,file.driver=$format";
1605     } elsif ($format) {
1606         $opts .= ",format=$format";
1607     }
1608
1609     my $cache_direct = 0;
1610
1611     if (my $cache = $drive->{cache}) {
1612         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1613     } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1614         $opts .= ",cache=none";
1615         $cache_direct = 1;
1616     }
1617
1618     # io_uring with cache mode writeback or writethrough on krbd will hang...
1619     my $rbd_no_io_uring = $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1620
1621     # io_uring with cache mode writeback or writethrough on LVM will hang...
1622     my $lvm_no_io_uring = $scfg && $scfg->{type} eq 'lvm' && !$cache_direct;
1623
1624     if (!$drive->{aio}) {
1625         if ($io_uring && !$rbd_no_io_uring && !$lvm_no_io_uring) {
1626             # io_uring supports all cache modes
1627             $opts .= ",aio=io_uring";
1628         } else {
1629             # aio native works only with O_DIRECT
1630             if($cache_direct) {
1631                 $opts .= ",aio=native";
1632             } else {
1633                 $opts .= ",aio=threads";
1634             }
1635         }
1636     }
1637
1638     if (!drive_is_cdrom($drive)) {
1639         my $detectzeroes;
1640         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1641             $detectzeroes = 'off';
1642         } elsif ($drive->{discard}) {
1643             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1644         } else {
1645             # This used to be our default with discard not being specified:
1646             $detectzeroes = 'on';
1647         }
1648
1649         # note: 'detect-zeroes' works per blockdev and we want it to persist
1650         # after the alloc-track is removed, so put it on 'file' directly
1651         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1652         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1653     }
1654
1655     if ($pbs_name) {
1656         $opts .= ",backing=$pbs_name";
1657         $opts .= ",auto-remove=on";
1658     }
1659
1660     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1661     my $file_param = "file";
1662     if ($pbs_name) {
1663         # non-rbd drivers require the underlying file to be a seperate block
1664         # node, so add a second .file indirection
1665         $file_param .= ".file" if !$is_rbd;
1666         $file_param .= ".filename";
1667     }
1668     my $pathinfo = $path ? "$file_param=$path," : '';
1669
1670     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1671 }
1672
1673 sub print_pbs_blockdev {
1674     my ($pbs_conf, $pbs_name) = @_;
1675     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1676     $blockdev .= ",repository=$pbs_conf->{repository}";
1677     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1678     $blockdev .= ",archive=$pbs_conf->{archive}";
1679     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1680     return $blockdev;
1681 }
1682
1683 sub print_netdevice_full {
1684     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1685
1686     my $device = $net->{model};
1687     if ($net->{model} eq 'virtio') {
1688          $device = 'virtio-net-pci';
1689      };
1690
1691     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1692     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1693     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1694         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1695         # and out of each queue plus one config interrupt and control vector queue
1696         my $vectors = $net->{queues} * 2 + 2;
1697         $tmpstr .= ",vectors=$vectors,mq=on";
1698     }
1699     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1700
1701     if (my $mtu = $net->{mtu}) {
1702         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1703             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1704             if ($mtu == 1) {
1705                  $mtu = $bridge_mtu;
1706             } elsif ($mtu < 576) {
1707                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1708             } elsif ($mtu > $bridge_mtu) {
1709                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1710             }
1711             $tmpstr .= ",host_mtu=$mtu";
1712         } else {
1713             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1714         }
1715     }
1716
1717     if ($use_old_bios_files) {
1718         my $romfile;
1719         if ($device eq 'virtio-net-pci') {
1720             $romfile = 'pxe-virtio.rom';
1721         } elsif ($device eq 'e1000') {
1722             $romfile = 'pxe-e1000.rom';
1723         } elsif ($device eq 'e1000e') {
1724             $romfile = 'pxe-e1000e.rom';
1725         } elsif ($device eq 'ne2k') {
1726             $romfile = 'pxe-ne2k_pci.rom';
1727         } elsif ($device eq 'pcnet') {
1728             $romfile = 'pxe-pcnet.rom';
1729         } elsif ($device eq 'rtl8139') {
1730             $romfile = 'pxe-rtl8139.rom';
1731         }
1732         $tmpstr .= ",romfile=$romfile" if $romfile;
1733     }
1734
1735     return $tmpstr;
1736 }
1737
1738 sub print_netdev_full {
1739     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1740
1741     my $i = '';
1742     if ($netid =~ m/^net(\d+)$/) {
1743         $i = int($1);
1744     }
1745
1746     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1747
1748     my $ifname = "tap${vmid}i$i";
1749
1750     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1751     die "interface name '$ifname' is too long (max 15 character)\n"
1752         if length($ifname) >= 16;
1753
1754     my $vhostparam = '';
1755     if (is_native($arch)) {
1756         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1757     }
1758
1759     my $vmname = $conf->{name} || "vm$vmid";
1760
1761     my $netdev = "";
1762     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1763
1764     if ($net->{bridge}) {
1765         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1766             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1767     } else {
1768         $netdev = "type=user,id=$netid,hostname=$vmname";
1769     }
1770
1771     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1772
1773     return $netdev;
1774 }
1775
1776 my $vga_map = {
1777     'cirrus' => 'cirrus-vga',
1778     'std' => 'VGA',
1779     'vmware' => 'vmware-svga',
1780     'virtio' => 'virtio-vga',
1781 };
1782
1783 sub print_vga_device {
1784     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1785
1786     my $type = $vga_map->{$vga->{type}};
1787     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1788         $type = 'virtio-gpu';
1789     }
1790     my $vgamem_mb = $vga->{memory};
1791
1792     my $max_outputs = '';
1793     if ($qxlnum) {
1794         $type = $id ? 'qxl' : 'qxl-vga';
1795
1796         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1797             # set max outputs so linux can have up to 4 qxl displays with one device
1798             if (min_version($machine_version, 4, 1)) {
1799                 $max_outputs = ",max_outputs=4";
1800             }
1801         }
1802     }
1803
1804     die "no devicetype for $vga->{type}\n" if !$type;
1805
1806     my $memory = "";
1807     if ($vgamem_mb) {
1808         if ($vga->{type} eq 'virtio') {
1809             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1810             $memory = ",max_hostmem=$bytes";
1811         } elsif ($qxlnum) {
1812             # from https://www.spice-space.org/multiple-monitors.html
1813             $memory = ",vgamem_mb=$vga->{memory}";
1814             my $ram = $vgamem_mb * 4;
1815             my $vram = $vgamem_mb * 2;
1816             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1817         } else {
1818             $memory = ",vgamem_mb=$vga->{memory}";
1819         }
1820     } elsif ($qxlnum && $id) {
1821         $memory = ",ram_size=67108864,vram_size=33554432";
1822     }
1823
1824     my $edidoff = "";
1825     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1826         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1827     }
1828
1829     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1830     my $vgaid = "vga" . ($id // '');
1831     my $pciaddr;
1832
1833     if ($q35 && $vgaid eq 'vga') {
1834         # the first display uses pcie.0 bus on q35 machines
1835         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1836     } else {
1837         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1838     }
1839
1840     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1841 }
1842
1843 sub parse_number_sets {
1844     my ($set) = @_;
1845     my $res = [];
1846     foreach my $part (split(/;/, $set)) {
1847         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1848             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1849             push @$res, [ $1, $2 ];
1850         } else {
1851             die "invalid range: $part\n";
1852         }
1853     }
1854     return $res;
1855 }
1856
1857 sub parse_numa {
1858     my ($data) = @_;
1859
1860     my $res = parse_property_string($numa_fmt, $data);
1861     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1862     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1863     return $res;
1864 }
1865
1866 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1867 sub parse_net {
1868     my ($data) = @_;
1869
1870     my $res = eval { parse_property_string($net_fmt, $data) };
1871     if ($@) {
1872         warn $@;
1873         return;
1874     }
1875     if (!defined($res->{macaddr})) {
1876         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1877         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1878     }
1879     return $res;
1880 }
1881
1882 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1883 sub parse_ipconfig {
1884     my ($data) = @_;
1885
1886     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1887     if ($@) {
1888         warn $@;
1889         return;
1890     }
1891
1892     if ($res->{gw} && !$res->{ip}) {
1893         warn 'gateway specified without specifying an IP address';
1894         return;
1895     }
1896     if ($res->{gw6} && !$res->{ip6}) {
1897         warn 'IPv6 gateway specified without specifying an IPv6 address';
1898         return;
1899     }
1900     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1901         warn 'gateway specified together with DHCP';
1902         return;
1903     }
1904     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1905         # gw6 + auto/dhcp
1906         warn "IPv6 gateway specified together with $res->{ip6} address";
1907         return;
1908     }
1909
1910     if (!$res->{ip} && !$res->{ip6}) {
1911         return { ip => 'dhcp', ip6 => 'dhcp' };
1912     }
1913
1914     return $res;
1915 }
1916
1917 sub print_net {
1918     my $net = shift;
1919
1920     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1921 }
1922
1923 sub add_random_macs {
1924     my ($settings) = @_;
1925
1926     foreach my $opt (keys %$settings) {
1927         next if $opt !~ m/^net(\d+)$/;
1928         my $net = parse_net($settings->{$opt});
1929         next if !$net;
1930         $settings->{$opt} = print_net($net);
1931     }
1932 }
1933
1934 sub vm_is_volid_owner {
1935     my ($storecfg, $vmid, $volid) = @_;
1936
1937     if ($volid !~  m|^/|) {
1938         my ($path, $owner);
1939         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1940         if ($owner && ($owner == $vmid)) {
1941             return 1;
1942         }
1943     }
1944
1945     return;
1946 }
1947
1948 sub vmconfig_register_unused_drive {
1949     my ($storecfg, $vmid, $conf, $drive) = @_;
1950
1951     if (drive_is_cloudinit($drive)) {
1952         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1953         warn $@ if $@;
1954     } elsif (!drive_is_cdrom($drive)) {
1955         my $volid = $drive->{file};
1956         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1957             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1958         }
1959     }
1960 }
1961
1962 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1963 my $smbios1_fmt = {
1964     uuid => {
1965         type => 'string',
1966         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1967         format_description => 'UUID',
1968         description => "Set SMBIOS1 UUID.",
1969         optional => 1,
1970     },
1971     version => {
1972         type => 'string',
1973         pattern => '[A-Za-z0-9+\/]+={0,2}',
1974         format_description => 'Base64 encoded string',
1975         description => "Set SMBIOS1 version.",
1976         optional => 1,
1977     },
1978     serial => {
1979         type => 'string',
1980         pattern => '[A-Za-z0-9+\/]+={0,2}',
1981         format_description => 'Base64 encoded string',
1982         description => "Set SMBIOS1 serial number.",
1983         optional => 1,
1984     },
1985     manufacturer => {
1986         type => 'string',
1987         pattern => '[A-Za-z0-9+\/]+={0,2}',
1988         format_description => 'Base64 encoded string',
1989         description => "Set SMBIOS1 manufacturer.",
1990         optional => 1,
1991     },
1992     product => {
1993         type => 'string',
1994         pattern => '[A-Za-z0-9+\/]+={0,2}',
1995         format_description => 'Base64 encoded string',
1996         description => "Set SMBIOS1 product ID.",
1997         optional => 1,
1998     },
1999     sku => {
2000         type => 'string',
2001         pattern => '[A-Za-z0-9+\/]+={0,2}',
2002         format_description => 'Base64 encoded string',
2003         description => "Set SMBIOS1 SKU string.",
2004         optional => 1,
2005     },
2006     family => {
2007         type => 'string',
2008         pattern => '[A-Za-z0-9+\/]+={0,2}',
2009         format_description => 'Base64 encoded string',
2010         description => "Set SMBIOS1 family string.",
2011         optional => 1,
2012     },
2013     base64 => {
2014         type => 'boolean',
2015         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2016         optional => 1,
2017     },
2018 };
2019
2020 sub parse_smbios1 {
2021     my ($data) = @_;
2022
2023     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2024     warn $@ if $@;
2025     return $res;
2026 }
2027
2028 sub print_smbios1 {
2029     my ($smbios1) = @_;
2030     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2031 }
2032
2033 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2034
2035 sub parse_watchdog {
2036     my ($value) = @_;
2037
2038     return if !$value;
2039
2040     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2041     warn $@ if $@;
2042     return $res;
2043 }
2044
2045 sub parse_guest_agent {
2046     my ($conf) = @_;
2047
2048     return {} if !defined($conf->{agent});
2049
2050     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2051     warn $@ if $@;
2052
2053     # if the agent is disabled ignore the other potentially set properties
2054     return {} if !$res->{enabled};
2055     return $res;
2056 }
2057
2058 sub get_qga_key {
2059     my ($conf, $key) = @_;
2060     return undef if !defined($conf->{agent});
2061
2062     my $agent = parse_guest_agent($conf);
2063     return $agent->{$key};
2064 }
2065
2066 sub parse_vga {
2067     my ($value) = @_;
2068
2069     return {} if !$value;
2070     my $res = eval { parse_property_string($vga_fmt, $value) };
2071     warn $@ if $@;
2072     return $res;
2073 }
2074
2075 sub parse_rng {
2076     my ($value) = @_;
2077
2078     return if !$value;
2079
2080     my $res = eval { parse_property_string($rng_fmt, $value) };
2081     warn $@ if $@;
2082     return $res;
2083 }
2084
2085 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2086 sub verify_usb_device {
2087     my ($value, $noerr) = @_;
2088
2089     return $value if parse_usb_device($value);
2090
2091     return if $noerr;
2092
2093     die "unable to parse usb device\n";
2094 }
2095
2096 # add JSON properties for create and set function
2097 sub json_config_properties {
2098     my $prop = shift;
2099
2100     foreach my $opt (keys %$confdesc) {
2101         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' ||
2102             $opt eq 'runningmachine' || $opt eq 'runningcpu';
2103         $prop->{$opt} = $confdesc->{$opt};
2104     }
2105
2106     return $prop;
2107 }
2108
2109 # return copy of $confdesc_cloudinit to generate documentation
2110 sub cloudinit_config_properties {
2111
2112     return dclone($confdesc_cloudinit);
2113 }
2114
2115 sub check_type {
2116     my ($key, $value) = @_;
2117
2118     die "unknown setting '$key'\n" if !$confdesc->{$key};
2119
2120     my $type = $confdesc->{$key}->{type};
2121
2122     if (!defined($value)) {
2123         die "got undefined value\n";
2124     }
2125
2126     if ($value =~ m/[\n\r]/) {
2127         die "property contains a line feed\n";
2128     }
2129
2130     if ($type eq 'boolean') {
2131         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2132         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2133         die "type check ('boolean') failed - got '$value'\n";
2134     } elsif ($type eq 'integer') {
2135         return int($1) if $value =~ m/^(\d+)$/;
2136         die "type check ('integer') failed - got '$value'\n";
2137     } elsif ($type eq 'number') {
2138         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2139         die "type check ('number') failed - got '$value'\n";
2140     } elsif ($type eq 'string') {
2141         if (my $fmt = $confdesc->{$key}->{format}) {
2142             PVE::JSONSchema::check_format($fmt, $value);
2143             return $value;
2144         }
2145         $value =~ s/^\"(.*)\"$/$1/;
2146         return $value;
2147     } else {
2148         die "internal error"
2149     }
2150 }
2151
2152 sub destroy_vm {
2153     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2154
2155     my $conf = PVE::QemuConfig->load_config($vmid);
2156
2157     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2158
2159     if ($conf->{template}) {
2160         # check if any base image is still used by a linked clone
2161         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2162                 my ($ds, $drive) = @_;
2163                 return if drive_is_cdrom($drive);
2164
2165                 my $volid = $drive->{file};
2166                 return if !$volid || $volid =~ m|^/|;
2167
2168                 die "base volume '$volid' is still in use by linked cloned\n"
2169                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2170
2171         });
2172     }
2173
2174     my $remove_owned_drive = sub {
2175         my ($ds, $drive) = @_;
2176         return if drive_is_cdrom($drive, 1);
2177
2178         my $volid = $drive->{file};
2179         return if !$volid || $volid =~ m|^/|;
2180
2181         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2182         return if !$path || !$owner || ($owner != $vmid);
2183
2184         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2185         warn "Could not remove disk '$volid', check manually: $@" if $@;
2186     };
2187
2188     # only remove disks owned by this VM (referenced in the config)
2189     my $include_opts = {
2190         include_unused => 1,
2191         extra_keys => ['vmstate'],
2192     };
2193     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2194
2195     for my $snap (values %{$conf->{snapshots}}) {
2196         next if !defined($snap->{vmstate});
2197         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2198         next if !defined($drive);
2199         $remove_owned_drive->('vmstate', $drive);
2200     }
2201
2202     if ($purge_unreferenced) { # also remove unreferenced disk
2203         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2204         PVE::Storage::foreach_volid($vmdisks, sub {
2205             my ($volid, $sid, $volname, $d) = @_;
2206             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2207             warn $@ if $@;
2208         });
2209     }
2210
2211     if (defined $replacement_conf) {
2212         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2213     } else {
2214         PVE::QemuConfig->destroy_config($vmid);
2215     }
2216 }
2217
2218 sub parse_vm_config {
2219     my ($filename, $raw) = @_;
2220
2221     return if !defined($raw);
2222
2223     my $res = {
2224         digest => Digest::SHA::sha1_hex($raw),
2225         snapshots => {},
2226         pending => {},
2227     };
2228
2229     $filename =~ m|/qemu-server/(\d+)\.conf$|
2230         || die "got strange filename '$filename'";
2231
2232     my $vmid = $1;
2233
2234     my $conf = $res;
2235     my $descr;
2236     my $section = '';
2237
2238     my @lines = split(/\n/, $raw);
2239     foreach my $line (@lines) {
2240         next if $line =~ m/^\s*$/;
2241
2242         if ($line =~ m/^\[PENDING\]\s*$/i) {
2243             $section = 'pending';
2244             if (defined($descr)) {
2245                 $descr =~ s/\s+$//;
2246                 $conf->{description} = $descr;
2247             }
2248             $descr = undef;
2249             $conf = $res->{$section} = {};
2250             next;
2251
2252         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2253             $section = $1;
2254             if (defined($descr)) {
2255                 $descr =~ s/\s+$//;
2256                 $conf->{description} = $descr;
2257             }
2258             $descr = undef;
2259             $conf = $res->{snapshots}->{$section} = {};
2260             next;
2261         }
2262
2263         if ($line =~ m/^\#(.*)\s*$/) {
2264             $descr = '' if !defined($descr);
2265             $descr .= PVE::Tools::decode_text($1) . "\n";
2266             next;
2267         }
2268
2269         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2270             $descr = '' if !defined($descr);
2271             $descr .= PVE::Tools::decode_text($2);
2272         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2273             $conf->{snapstate} = $1;
2274         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2275             my $key = $1;
2276             my $value = $2;
2277             $conf->{$key} = $value;
2278         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2279             my $value = $1;
2280             if ($section eq 'pending') {
2281                 $conf->{delete} = $value; # we parse this later
2282             } else {
2283                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2284             }
2285         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2286             my $key = $1;
2287             my $value = $2;
2288             eval { $value = check_type($key, $value); };
2289             if ($@) {
2290                 warn "vm $vmid - unable to parse value of '$key' - $@";
2291             } else {
2292                 $key = 'ide2' if $key eq 'cdrom';
2293                 my $fmt = $confdesc->{$key}->{format};
2294                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2295                     my $v = parse_drive($key, $value);
2296                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2297                         $v->{file} = $volid;
2298                         $value = print_drive($v);
2299                     } else {
2300                         warn "vm $vmid - unable to parse value of '$key'\n";
2301                         next;
2302                     }
2303                 }
2304
2305                 $conf->{$key} = $value;
2306             }
2307         } else {
2308             warn "vm $vmid - unable to parse config: $line\n";
2309         }
2310     }
2311
2312     if (defined($descr)) {
2313         $descr =~ s/\s+$//;
2314         $conf->{description} = $descr;
2315     }
2316     delete $res->{snapstate}; # just to be sure
2317
2318     return $res;
2319 }
2320
2321 sub write_vm_config {
2322     my ($filename, $conf) = @_;
2323
2324     delete $conf->{snapstate}; # just to be sure
2325
2326     if ($conf->{cdrom}) {
2327         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2328         $conf->{ide2} = $conf->{cdrom};
2329         delete $conf->{cdrom};
2330     }
2331
2332     # we do not use 'smp' any longer
2333     if ($conf->{sockets}) {
2334         delete $conf->{smp};
2335     } elsif ($conf->{smp}) {
2336         $conf->{sockets} = $conf->{smp};
2337         delete $conf->{cores};
2338         delete $conf->{smp};
2339     }
2340
2341     my $used_volids = {};
2342
2343     my $cleanup_config = sub {
2344         my ($cref, $pending, $snapname) = @_;
2345
2346         foreach my $key (keys %$cref) {
2347             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2348                 $key eq 'snapstate' || $key eq 'pending';
2349             my $value = $cref->{$key};
2350             if ($key eq 'delete') {
2351                 die "propertry 'delete' is only allowed in [PENDING]\n"
2352                     if !$pending;
2353                 # fixme: check syntax?
2354                 next;
2355             }
2356             eval { $value = check_type($key, $value); };
2357             die "unable to parse value of '$key' - $@" if $@;
2358
2359             $cref->{$key} = $value;
2360
2361             if (!$snapname && is_valid_drivename($key)) {
2362                 my $drive = parse_drive($key, $value);
2363                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2364             }
2365         }
2366     };
2367
2368     &$cleanup_config($conf);
2369
2370     &$cleanup_config($conf->{pending}, 1);
2371
2372     foreach my $snapname (keys %{$conf->{snapshots}}) {
2373         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2374         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2375     }
2376
2377     # remove 'unusedX' settings if we re-add a volume
2378     foreach my $key (keys %$conf) {
2379         my $value = $conf->{$key};
2380         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2381             delete $conf->{$key};
2382         }
2383     }
2384
2385     my $generate_raw_config = sub {
2386         my ($conf, $pending) = @_;
2387
2388         my $raw = '';
2389
2390         # add description as comment to top of file
2391         if (defined(my $descr = $conf->{description})) {
2392             if ($descr) {
2393                 foreach my $cl (split(/\n/, $descr)) {
2394                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2395                 }
2396             } else {
2397                 $raw .= "#\n" if $pending;
2398             }
2399         }
2400
2401         foreach my $key (sort keys %$conf) {
2402             next if $key =~ /^(digest|description|pending|snapshots)$/;
2403             $raw .= "$key: $conf->{$key}\n";
2404         }
2405         return $raw;
2406     };
2407
2408     my $raw = &$generate_raw_config($conf);
2409
2410     if (scalar(keys %{$conf->{pending}})){
2411         $raw .= "\n[PENDING]\n";
2412         $raw .= &$generate_raw_config($conf->{pending}, 1);
2413     }
2414
2415     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2416         $raw .= "\n[$snapname]\n";
2417         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2418     }
2419
2420     return $raw;
2421 }
2422
2423 sub load_defaults {
2424
2425     my $res = {};
2426
2427     # we use static defaults from our JSON schema configuration
2428     foreach my $key (keys %$confdesc) {
2429         if (defined(my $default = $confdesc->{$key}->{default})) {
2430             $res->{$key} = $default;
2431         }
2432     }
2433
2434     return $res;
2435 }
2436
2437 sub config_list {
2438     my $vmlist = PVE::Cluster::get_vmlist();
2439     my $res = {};
2440     return $res if !$vmlist || !$vmlist->{ids};
2441     my $ids = $vmlist->{ids};
2442     my $nodename = nodename();
2443
2444     foreach my $vmid (keys %$ids) {
2445         my $d = $ids->{$vmid};
2446         next if !$d->{node} || $d->{node} ne $nodename;
2447         next if !$d->{type} || $d->{type} ne 'qemu';
2448         $res->{$vmid}->{exists} = 1;
2449     }
2450     return $res;
2451 }
2452
2453 # test if VM uses local resources (to prevent migration)
2454 sub check_local_resources {
2455     my ($conf, $noerr) = @_;
2456
2457     my @loc_res = ();
2458
2459     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2460     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2461
2462     push @loc_res, "ivshmem" if $conf->{ivshmem};
2463
2464     foreach my $k (keys %$conf) {
2465         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2466         # sockets are safe: they will recreated be on the target side post-migrate
2467         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2468         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2469     }
2470
2471     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2472
2473     return \@loc_res;
2474 }
2475
2476 # check if used storages are available on all nodes (use by migrate)
2477 sub check_storage_availability {
2478     my ($storecfg, $conf, $node) = @_;
2479
2480     PVE::QemuConfig->foreach_volume($conf, sub {
2481         my ($ds, $drive) = @_;
2482
2483         my $volid = $drive->{file};
2484         return if !$volid;
2485
2486         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2487         return if !$sid;
2488
2489         # check if storage is available on both nodes
2490         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2491         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2492
2493         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2494
2495         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2496             if !$scfg->{content}->{$vtype};
2497    });
2498 }
2499
2500 # list nodes where all VM images are available (used by has_feature API)
2501 sub shared_nodes {
2502     my ($conf, $storecfg) = @_;
2503
2504     my $nodelist = PVE::Cluster::get_nodelist();
2505     my $nodehash = { map { $_ => 1 } @$nodelist };
2506     my $nodename = nodename();
2507
2508     PVE::QemuConfig->foreach_volume($conf, sub {
2509         my ($ds, $drive) = @_;
2510
2511         my $volid = $drive->{file};
2512         return if !$volid;
2513
2514         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2515         if ($storeid) {
2516             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2517             if ($scfg->{disable}) {
2518                 $nodehash = {};
2519             } elsif (my $avail = $scfg->{nodes}) {
2520                 foreach my $node (keys %$nodehash) {
2521                     delete $nodehash->{$node} if !$avail->{$node};
2522                 }
2523             } elsif (!$scfg->{shared}) {
2524                 foreach my $node (keys %$nodehash) {
2525                     delete $nodehash->{$node} if $node ne $nodename
2526                 }
2527             }
2528         }
2529     });
2530
2531     return $nodehash
2532 }
2533
2534 sub check_local_storage_availability {
2535     my ($conf, $storecfg) = @_;
2536
2537     my $nodelist = PVE::Cluster::get_nodelist();
2538     my $nodehash = { map { $_ => {} } @$nodelist };
2539
2540     PVE::QemuConfig->foreach_volume($conf, sub {
2541         my ($ds, $drive) = @_;
2542
2543         my $volid = $drive->{file};
2544         return if !$volid;
2545
2546         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2547         if ($storeid) {
2548             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2549
2550             if ($scfg->{disable}) {
2551                 foreach my $node (keys %$nodehash) {
2552                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2553                 }
2554             } elsif (my $avail = $scfg->{nodes}) {
2555                 foreach my $node (keys %$nodehash) {
2556                     if (!$avail->{$node}) {
2557                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2558                     }
2559                 }
2560             }
2561         }
2562     });
2563
2564     foreach my $node (values %$nodehash) {
2565         if (my $unavail = $node->{unavailable_storages}) {
2566             $node->{unavailable_storages} = [ sort keys %$unavail ];
2567         }
2568     }
2569
2570     return $nodehash
2571 }
2572
2573 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2574 sub check_running {
2575     my ($vmid, $nocheck, $node) = @_;
2576
2577     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2578     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2579 }
2580
2581 sub vzlist {
2582
2583     my $vzlist = config_list();
2584
2585     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2586
2587     while (defined(my $de = $fd->read)) {
2588         next if $de !~ m/^(\d+)\.pid$/;
2589         my $vmid = $1;
2590         next if !defined($vzlist->{$vmid});
2591         if (my $pid = check_running($vmid)) {
2592             $vzlist->{$vmid}->{pid} = $pid;
2593         }
2594     }
2595
2596     return $vzlist;
2597 }
2598
2599 our $vmstatus_return_properties = {
2600     vmid => get_standard_option('pve-vmid'),
2601     status => {
2602         description => "Qemu process status.",
2603         type => 'string',
2604         enum => ['stopped', 'running'],
2605     },
2606     maxmem => {
2607         description => "Maximum memory in bytes.",
2608         type => 'integer',
2609         optional => 1,
2610         renderer => 'bytes',
2611     },
2612     maxdisk => {
2613         description => "Root disk size in bytes.",
2614         type => 'integer',
2615         optional => 1,
2616         renderer => 'bytes',
2617     },
2618     name => {
2619         description => "VM name.",
2620         type => 'string',
2621         optional => 1,
2622     },
2623     qmpstatus => {
2624         description => "Qemu QMP agent status.",
2625         type => 'string',
2626         optional => 1,
2627     },
2628     pid => {
2629         description => "PID of running qemu process.",
2630         type => 'integer',
2631         optional => 1,
2632     },
2633     uptime => {
2634         description => "Uptime.",
2635         type => 'integer',
2636         optional => 1,
2637         renderer => 'duration',
2638     },
2639     cpus => {
2640         description => "Maximum usable CPUs.",
2641         type => 'number',
2642         optional => 1,
2643     },
2644     lock => {
2645         description => "The current config lock, if any.",
2646         type => 'string',
2647         optional => 1,
2648     },
2649     tags => {
2650         description  => "The current configured tags, if any",
2651         type => 'string',
2652         optional => 1,
2653     },
2654     'running-machine' => {
2655         description => "The currently running machine type (if running).",
2656         type => 'string',
2657         optional => 1,
2658     },
2659     'running-qemu' => {
2660         description => "The currently running QEMU version (if running).",
2661         type => 'string',
2662         optional => 1,
2663     },
2664 };
2665
2666 my $last_proc_pid_stat;
2667
2668 # get VM status information
2669 # This must be fast and should not block ($full == false)
2670 # We only query KVM using QMP if $full == true (this can be slow)
2671 sub vmstatus {
2672     my ($opt_vmid, $full) = @_;
2673
2674     my $res = {};
2675
2676     my $storecfg = PVE::Storage::config();
2677
2678     my $list = vzlist();
2679     my $defaults = load_defaults();
2680
2681     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2682
2683     my $cpucount = $cpuinfo->{cpus} || 1;
2684
2685     foreach my $vmid (keys %$list) {
2686         next if $opt_vmid && ($vmid ne $opt_vmid);
2687
2688         my $conf = PVE::QemuConfig->load_config($vmid);
2689
2690         my $d = { vmid => int($vmid) };
2691         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2692
2693         # fixme: better status?
2694         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2695
2696         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2697         if (defined($size)) {
2698             $d->{disk} = 0; # no info available
2699             $d->{maxdisk} = $size;
2700         } else {
2701             $d->{disk} = 0;
2702             $d->{maxdisk} = 0;
2703         }
2704
2705         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2706             * ($conf->{cores} || $defaults->{cores});
2707         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2708         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2709
2710         $d->{name} = $conf->{name} || "VM $vmid";
2711         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2712             : $defaults->{memory}*(1024*1024);
2713
2714         if ($conf->{balloon}) {
2715             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2716             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2717                 : $defaults->{shares};
2718         }
2719
2720         $d->{uptime} = 0;
2721         $d->{cpu} = 0;
2722         $d->{mem} = 0;
2723
2724         $d->{netout} = 0;
2725         $d->{netin} = 0;
2726
2727         $d->{diskread} = 0;
2728         $d->{diskwrite} = 0;
2729
2730         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2731
2732         $d->{serial} = 1 if conf_has_serial($conf);
2733         $d->{lock} = $conf->{lock} if $conf->{lock};
2734         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2735
2736         $res->{$vmid} = $d;
2737     }
2738
2739     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2740     foreach my $dev (keys %$netdev) {
2741         next if $dev !~ m/^tap([1-9]\d*)i/;
2742         my $vmid = $1;
2743         my $d = $res->{$vmid};
2744         next if !$d;
2745
2746         $d->{netout} += $netdev->{$dev}->{receive};
2747         $d->{netin} += $netdev->{$dev}->{transmit};
2748
2749         if ($full) {
2750             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2751             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2752         }
2753
2754     }
2755
2756     my $ctime = gettimeofday;
2757
2758     foreach my $vmid (keys %$list) {
2759
2760         my $d = $res->{$vmid};
2761         my $pid = $d->{pid};
2762         next if !$pid;
2763
2764         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2765         next if !$pstat; # not running
2766
2767         my $used = $pstat->{utime} + $pstat->{stime};
2768
2769         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2770
2771         if ($pstat->{vsize}) {
2772             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2773         }
2774
2775         my $old = $last_proc_pid_stat->{$pid};
2776         if (!$old) {
2777             $last_proc_pid_stat->{$pid} = {
2778                 time => $ctime,
2779                 used => $used,
2780                 cpu => 0,
2781             };
2782             next;
2783         }
2784
2785         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2786
2787         if ($dtime > 1000) {
2788             my $dutime = $used -  $old->{used};
2789
2790             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2791             $last_proc_pid_stat->{$pid} = {
2792                 time => $ctime,
2793                 used => $used,
2794                 cpu => $d->{cpu},
2795             };
2796         } else {
2797             $d->{cpu} = $old->{cpu};
2798         }
2799     }
2800
2801     return $res if !$full;
2802
2803     my $qmpclient = PVE::QMPClient->new();
2804
2805     my $ballooncb = sub {
2806         my ($vmid, $resp) = @_;
2807
2808         my $info = $resp->{'return'};
2809         return if !$info->{max_mem};
2810
2811         my $d = $res->{$vmid};
2812
2813         # use memory assigned to VM
2814         $d->{maxmem} = $info->{max_mem};
2815         $d->{balloon} = $info->{actual};
2816
2817         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2818             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2819             $d->{freemem} = $info->{free_mem};
2820         }
2821
2822         $d->{ballooninfo} = $info;
2823     };
2824
2825     my $blockstatscb = sub {
2826         my ($vmid, $resp) = @_;
2827         my $data = $resp->{'return'} || [];
2828         my $totalrdbytes = 0;
2829         my $totalwrbytes = 0;
2830
2831         for my $blockstat (@$data) {
2832             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2833             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2834
2835             $blockstat->{device} =~ s/drive-//;
2836             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2837         }
2838         $res->{$vmid}->{diskread} = $totalrdbytes;
2839         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2840     };
2841
2842     my $machinecb = sub {
2843         my ($vmid, $resp) = @_;
2844         my $data = $resp->{'return'} || [];
2845
2846         $res->{$vmid}->{'running-machine'} =
2847             PVE::QemuServer::Machine::current_from_query_machines($data);
2848     };
2849
2850     my $versioncb = sub {
2851         my ($vmid, $resp) = @_;
2852         my $data = $resp->{'return'} // {};
2853         my $version = 'unknown';
2854
2855         if (my $v = $data->{qemu}) {
2856             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
2857         }
2858
2859         $res->{$vmid}->{'running-qemu'} = $version;
2860     };
2861
2862     my $statuscb = sub {
2863         my ($vmid, $resp) = @_;
2864
2865         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2866         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
2867         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
2868         # this fails if ballon driver is not loaded, so this must be
2869         # the last commnand (following command are aborted if this fails).
2870         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2871
2872         my $status = 'unknown';
2873         if (!defined($status = $resp->{'return'}->{status})) {
2874             warn "unable to get VM status\n";
2875             return;
2876         }
2877
2878         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2879     };
2880
2881     foreach my $vmid (keys %$list) {
2882         next if $opt_vmid && ($vmid ne $opt_vmid);
2883         next if !$res->{$vmid}->{pid}; # not running
2884         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2885     }
2886
2887     $qmpclient->queue_execute(undef, 2);
2888
2889     foreach my $vmid (keys %$list) {
2890         next if $opt_vmid && ($vmid ne $opt_vmid);
2891         next if !$res->{$vmid}->{pid}; #not running
2892
2893         # we can't use the $qmpclient since it might have already aborted on
2894         # 'query-balloon', but this might also fail for older versions...
2895         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
2896         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
2897     }
2898
2899     foreach my $vmid (keys %$list) {
2900         next if $opt_vmid && ($vmid ne $opt_vmid);
2901         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2902     }
2903
2904     return $res;
2905 }
2906
2907 sub conf_has_serial {
2908     my ($conf) = @_;
2909
2910     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2911         if ($conf->{"serial$i"}) {
2912             return 1;
2913         }
2914     }
2915
2916     return 0;
2917 }
2918
2919 sub conf_has_audio {
2920     my ($conf, $id) = @_;
2921
2922     $id //= 0;
2923     my $audio = $conf->{"audio$id"};
2924     return if !defined($audio);
2925
2926     my $audioproperties = parse_property_string($audio_fmt, $audio);
2927     my $audiodriver = $audioproperties->{driver} // 'spice';
2928
2929     return {
2930         dev => $audioproperties->{device},
2931         dev_id => "audiodev$id",
2932         backend => $audiodriver,
2933         backend_id => "$audiodriver-backend${id}",
2934     };
2935 }
2936
2937 sub audio_devs {
2938     my ($audio, $audiopciaddr, $machine_version) = @_;
2939
2940     my $devs = [];
2941
2942     my $id = $audio->{dev_id};
2943     my $audiodev = "";
2944     if (min_version($machine_version, 4, 2)) {
2945         $audiodev = ",audiodev=$audio->{backend_id}";
2946     }
2947
2948     if ($audio->{dev} eq 'AC97') {
2949         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
2950     } elsif ($audio->{dev} =~ /intel\-hda$/) {
2951         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
2952         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
2953         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
2954     } else {
2955         die "unkown audio device '$audio->{dev}', implement me!";
2956     }
2957
2958     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
2959
2960     return $devs;
2961 }
2962
2963 sub vga_conf_has_spice {
2964     my ($vga) = @_;
2965
2966     my $vgaconf = parse_vga($vga);
2967     my $vgatype = $vgaconf->{type};
2968     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
2969
2970     return $1 || 1;
2971 }
2972
2973 sub is_native($) {
2974     my ($arch) = @_;
2975     return get_host_arch() eq $arch;
2976 }
2977
2978 sub get_vm_arch {
2979     my ($conf) = @_;
2980     return $conf->{arch} // get_host_arch();
2981 }
2982
2983 my $default_machines = {
2984     x86_64 => 'pc',
2985     aarch64 => 'virt',
2986 };
2987
2988 sub get_installed_machine_version {
2989     my ($kvmversion) = @_;
2990     $kvmversion = kvm_user_version() if !defined($kvmversion);
2991     $kvmversion =~ m/^(\d+\.\d+)/;
2992     return $1;
2993 }
2994
2995 sub windows_get_pinned_machine_version {
2996     my ($machine, $base_version, $kvmversion) = @_;
2997
2998     my $pin_version = $base_version;
2999     if (!defined($base_version) ||
3000         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
3001     ) {
3002         $pin_version = get_installed_machine_version($kvmversion);
3003     }
3004     if (!$machine || $machine eq 'pc') {
3005         $machine = "pc-i440fx-$pin_version";
3006     } elsif ($machine eq 'q35') {
3007         $machine = "pc-q35-$pin_version";
3008     } elsif ($machine eq 'virt') {
3009         $machine = "virt-$pin_version";
3010     } else {
3011         warn "unknown machine type '$machine', not touching that!\n";
3012     }
3013
3014     return $machine;
3015 }
3016
3017 sub get_vm_machine {
3018     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3019
3020     my $machine = $forcemachine || $conf->{machine};
3021
3022     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3023         $kvmversion //= kvm_user_version();
3024         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3025         # layout which confuses windows quite a bit and may result in various regressions..
3026         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3027         if (windows_version($conf->{ostype})) {
3028             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3029         }
3030         $arch //= 'x86_64';
3031         $machine ||= $default_machines->{$arch};
3032         if ($add_pve_version) {
3033             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3034             $machine .= "+pve$pvever";
3035         }
3036     }
3037
3038     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3039         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3040         $machine = $1 if $is_pxe;
3041
3042         # for version-pinned machines that do not include a pve-version (e.g.
3043         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3044         $machine .= '+pve0';
3045
3046         $machine .= '.pxe' if $is_pxe;
3047     }
3048
3049     return $machine;
3050 }
3051
3052 sub get_ovmf_files($) {
3053     my ($arch) = @_;
3054
3055     my $ovmf = $OVMF->{$arch}
3056         or die "no OVMF images known for architecture '$arch'\n";
3057
3058     return @$ovmf;
3059 }
3060
3061 my $Arch2Qemu = {
3062     aarch64 => '/usr/bin/qemu-system-aarch64',
3063     x86_64 => '/usr/bin/qemu-system-x86_64',
3064 };
3065 sub get_command_for_arch($) {
3066     my ($arch) = @_;
3067     return '/usr/bin/kvm' if is_native($arch);
3068
3069     my $cmd = $Arch2Qemu->{$arch}
3070         or die "don't know how to emulate architecture '$arch'\n";
3071     return $cmd;
3072 }
3073
3074 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3075 # to use in a QEMU command line (-cpu element), first array_intersect the result
3076 # of query_supported_ with query_understood_. This is necessary because:
3077 #
3078 # a) query_understood_ returns flags the host cannot use and
3079 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3080 #    flags, but CPU settings - with most of them being flags. Those settings
3081 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3082 #
3083 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3084 # expensive. If you need the value returned from this, you can get it much
3085 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3086 # $accel being 'kvm' or 'tcg'.
3087 #
3088 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3089 # changes, automatically populating pmxcfs.
3090 #
3091 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3092 # since kvm and tcg machines support different flags
3093 #
3094 sub query_supported_cpu_flags {
3095     my ($arch) = @_;
3096
3097     $arch //= get_host_arch();
3098     my $default_machine = $default_machines->{$arch};
3099
3100     my $flags = {};
3101
3102     # FIXME: Once this is merged, the code below should work for ARM as well:
3103     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3104     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3105         $arch eq "aarch64";
3106
3107     my $kvm_supported = defined(kvm_version());
3108     my $qemu_cmd = get_command_for_arch($arch);
3109     my $fakevmid = -1;
3110     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3111
3112     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3113     my $query_supported_run_qemu = sub {
3114         my ($kvm) = @_;
3115
3116         my $flags = {};
3117         my $cmd = [
3118             $qemu_cmd,
3119             '-machine', $default_machine,
3120             '-display', 'none',
3121             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3122             '-mon', 'chardev=qmp,mode=control',
3123             '-pidfile', $pidfile,
3124             '-S', '-daemonize'
3125         ];
3126
3127         if (!$kvm) {
3128             push @$cmd, '-accel', 'tcg';
3129         }
3130
3131         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3132         die "QEMU flag querying VM exited with code " . $rc if $rc;
3133
3134         eval {
3135             my $cmd_result = mon_cmd(
3136                 $fakevmid,
3137                 'query-cpu-model-expansion',
3138                 type => 'full',
3139                 model => { name => 'host' }
3140             );
3141
3142             my $props = $cmd_result->{model}->{props};
3143             foreach my $prop (keys %$props) {
3144                 next if $props->{$prop} ne '1';
3145                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3146                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3147                 # We only keep those with underscores, to match /proc/cpuinfo
3148                 $prop =~ s/\.|-/_/g;
3149                 $flags->{$prop} = 1;
3150             }
3151         };
3152         my $err = $@;
3153
3154         # force stop with 10 sec timeout and 'nocheck'
3155         # always stop, even if QMP failed
3156         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3157
3158         die $err if $err;
3159
3160         return [ sort keys %$flags ];
3161     };
3162
3163     # We need to query QEMU twice, since KVM and TCG have different supported flags
3164     PVE::QemuConfig->lock_config($fakevmid, sub {
3165         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3166         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3167
3168         if ($kvm_supported) {
3169             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3170             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3171         }
3172     });
3173
3174     return $flags;
3175 }
3176
3177 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3178 my $understood_cpu_flag_dir = "/usr/share/kvm";
3179 sub query_understood_cpu_flags {
3180     my $arch = get_host_arch();
3181     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3182
3183     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3184         if ! -e $filepath;
3185
3186     my $raw = file_get_contents($filepath);
3187     $raw =~ s/^\s+|\s+$//g;
3188     my @flags = split(/\s+/, $raw);
3189
3190     return \@flags;
3191 }
3192
3193 my sub get_cpuunits {
3194     my ($conf) = @_;
3195     return $conf->{cpuunits} // (PVE::CGroup::cgroup_mode() == 2 ? 100 : 1024);
3196 }
3197 sub config_to_command {
3198     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3199         $pbs_backing) = @_;
3200
3201     my $cmd = [];
3202     my $globalFlags = [];
3203     my $machineFlags = [];
3204     my $rtcFlags = [];
3205     my $devices = [];
3206     my $pciaddr = '';
3207     my $bridges = {};
3208     my $ostype = $conf->{ostype};
3209     my $winversion = windows_version($ostype);
3210     my $kvm = $conf->{kvm};
3211     my $nodename = nodename();
3212
3213     my $arch = get_vm_arch($conf);
3214     my $kvm_binary = get_command_for_arch($arch);
3215     my $kvmver = kvm_user_version($kvm_binary);
3216
3217     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3218         $kvmver //= "undefined";
3219         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3220     }
3221
3222     my $add_pve_version = min_version($kvmver, 4, 1);
3223
3224     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3225     my $machine_version = extract_version($machine_type, $kvmver);
3226     $kvm //= 1 if is_native($arch);
3227
3228     $machine_version =~ m/(\d+)\.(\d+)/;
3229     my ($machine_major, $machine_minor) = ($1, $2);
3230
3231     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3232         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3233     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3234         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3235             ." please upgrade node '$nodename'\n"
3236     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3237         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3238         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3239             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3240             ." node '$nodename'\n";
3241     }
3242
3243     # if a specific +pve version is required for a feature, use $version_guard
3244     # instead of min_version to allow machines to be run with the minimum
3245     # required version
3246     my $required_pve_version = 0;
3247     my $version_guard = sub {
3248         my ($major, $minor, $pve) = @_;
3249         return 0 if !min_version($machine_version, $major, $minor, $pve);
3250         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3251         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3252         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3253         return 1;
3254     };
3255
3256     if ($kvm && !defined kvm_version()) {
3257         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3258             ." or enable in BIOS.\n";
3259     }
3260
3261     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3262     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3263     my $use_old_bios_files = undef;
3264     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3265
3266     my $cpuunits = get_cpuunits($conf);
3267
3268     push @$cmd, $kvm_binary;
3269
3270     push @$cmd, '-id', $vmid;
3271
3272     my $vmname = $conf->{name} || "vm$vmid";
3273
3274     push @$cmd, '-name', $vmname;
3275
3276     push @$cmd, '-no-shutdown';
3277
3278     my $use_virtio = 0;
3279
3280     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3281     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3282     push @$cmd, '-mon', "chardev=qmp,mode=control";
3283
3284     if (min_version($machine_version, 2, 12)) {
3285         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3286         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3287     }
3288
3289     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3290
3291     push @$cmd, '-daemonize';
3292
3293     if ($conf->{smbios1}) {
3294         my $smbios_conf = parse_smbios1($conf->{smbios1});
3295         if ($smbios_conf->{base64}) {
3296             # Do not pass base64 flag to qemu
3297             delete $smbios_conf->{base64};
3298             my $smbios_string = "";
3299             foreach my $key (keys %$smbios_conf) {
3300                 my $value;
3301                 if ($key eq "uuid") {
3302                     $value = $smbios_conf->{uuid}
3303                 } else {
3304                     $value = decode_base64($smbios_conf->{$key});
3305                 }
3306                 # qemu accepts any binary data, only commas need escaping by double comma
3307                 $value =~ s/,/,,/g;
3308                 $smbios_string .= "," . $key . "=" . $value if $value;
3309             }
3310             push @$cmd, '-smbios', "type=1" . $smbios_string;
3311         } else {
3312             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3313         }
3314     }
3315
3316     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3317         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3318         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3319
3320         my ($path, $format);
3321         my $read_only_str = '';
3322         if (my $efidisk = $conf->{efidisk0}) {
3323             my $d = parse_drive('efidisk0', $efidisk);
3324             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3325             $format = $d->{format};
3326             if ($storeid) {
3327                 $path = PVE::Storage::path($storecfg, $d->{file});
3328                 if (!defined($format)) {
3329                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3330                     $format = qemu_img_format($scfg, $volname);
3331                 }
3332             } else {
3333                 $path = $d->{file};
3334                 die "efidisk format must be specified\n"
3335                     if !defined($format);
3336             }
3337
3338             $read_only_str = ',readonly=on' if drive_is_read_only($conf, $d);
3339         } else {
3340             warn "no efidisk configured! Using temporary efivars disk.\n";
3341             $path = "/tmp/$vmid-ovmf.fd";
3342             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3343             $format = 'raw';
3344         }
3345
3346         my $size_str = "";
3347
3348         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3349             $size_str = ",size=" . (-s $ovmf_vars);
3350         }
3351
3352         # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3353         my $cache = "";
3354         if ($path =~ m/^rbd:/) {
3355                 $cache = ',cache=writeback';
3356                 $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3357         }
3358
3359         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code";
3360         push @$cmd, '-drive', "if=pflash,unit=1$cache,format=$format,id=drive-efidisk0$size_str,file=${path}${read_only_str}";
3361     }
3362
3363     # load q35 config
3364     if ($q35) {
3365         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3366         if (min_version($machine_version, 4, 0)) {
3367             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3368         } else {
3369             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3370         }
3371     }
3372
3373     if ($conf->{vmgenid}) {
3374         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3375     }
3376
3377     # add usb controllers
3378     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3379         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3380     push @$devices, @usbcontrollers if @usbcontrollers;
3381     my $vga = parse_vga($conf->{vga});
3382
3383     my $qxlnum = vga_conf_has_spice($conf->{vga});
3384     $vga->{type} = 'qxl' if $qxlnum;
3385
3386     if (!$vga->{type}) {
3387         if ($arch eq 'aarch64') {
3388             $vga->{type} = 'virtio';
3389         } elsif (min_version($machine_version, 2, 9)) {
3390             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3391         } else {
3392             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3393         }
3394     }
3395
3396     # enable absolute mouse coordinates (needed by vnc)
3397     my $tablet;
3398     if (defined($conf->{tablet})) {
3399         $tablet = $conf->{tablet};
3400     } else {
3401         $tablet = $defaults->{tablet};
3402         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3403         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3404     }
3405
3406     if ($tablet) {
3407         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3408         my $kbd = print_keyboarddevice_full($conf, $arch);
3409         push @$devices, '-device', $kbd if defined($kbd);
3410     }
3411
3412     my $bootorder = device_bootorder($conf);
3413
3414     # host pci device passthrough
3415     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3416         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3417
3418     # usb devices
3419     my $usb_dev_features = {};
3420     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3421
3422     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3423         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3424     push @$devices, @usbdevices if @usbdevices;
3425
3426     # serial devices
3427     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3428         if (my $path = $conf->{"serial$i"}) {
3429             if ($path eq 'socket') {
3430                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3431                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3432                 # On aarch64, serial0 is the UART device. Qemu only allows
3433                 # connecting UART devices via the '-serial' command line, as
3434                 # the device has a fixed slot on the hardware...
3435                 if ($arch eq 'aarch64' && $i == 0) {
3436                     push @$devices, '-serial', "chardev:serial$i";
3437                 } else {
3438                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3439                 }
3440             } else {
3441                 die "no such serial device\n" if ! -c $path;
3442                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3443                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3444             }
3445         }
3446     }
3447
3448     # parallel devices
3449     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3450         if (my $path = $conf->{"parallel$i"}) {
3451             die "no such parallel device\n" if ! -c $path;
3452             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3453             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3454             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3455         }
3456     }
3457
3458     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3459         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3460         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3461         push @$devices, @$audio_devs;
3462     }
3463
3464     my $sockets = 1;
3465     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3466     $sockets = $conf->{sockets} if  $conf->{sockets};
3467
3468     my $cores = $conf->{cores} || 1;
3469
3470     my $maxcpus = $sockets * $cores;
3471
3472     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3473
3474     my $allowed_vcpus = $cpuinfo->{cpus};
3475
3476     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3477         if ($allowed_vcpus < $maxcpus);
3478
3479     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3480
3481         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3482         for (my $i = 2; $i <= $vcpus; $i++)  {
3483             my $cpustr = print_cpu_device($conf,$i);
3484             push @$cmd, '-device', $cpustr;
3485         }
3486
3487     } else {
3488
3489         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3490     }
3491     push @$cmd, '-nodefaults';
3492
3493     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3494
3495     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3496
3497     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3498
3499     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3500         push @$devices, '-device', print_vga_device(
3501             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3502         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3503         push @$cmd,  '-vnc', "unix:$socket,password=on";
3504     } else {
3505         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3506         push @$cmd, '-nographic';
3507     }
3508
3509     # time drift fix
3510     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3511     my $useLocaltime = $conf->{localtime};
3512
3513     if ($winversion >= 5) { # windows
3514         $useLocaltime = 1 if !defined($conf->{localtime});
3515
3516         # use time drift fix when acpi is enabled
3517         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3518             $tdf = 1 if !defined($conf->{tdf});
3519         }
3520     }
3521
3522     if ($winversion >= 6) {
3523         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3524         push @$cmd, '-no-hpet';
3525     }
3526
3527     push @$rtcFlags, 'driftfix=slew' if $tdf;
3528
3529     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3530         push @$rtcFlags, "base=$conf->{startdate}";
3531     } elsif ($useLocaltime) {
3532         push @$rtcFlags, 'base=localtime';
3533     }
3534
3535     if ($forcecpu) {
3536         push @$cmd, '-cpu', $forcecpu;
3537     } else {
3538         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3539     }
3540
3541     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3542
3543     push @$cmd, '-S' if $conf->{freeze};
3544
3545     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3546
3547     my $guest_agent = parse_guest_agent($conf);
3548
3549     if ($guest_agent->{enabled}) {
3550         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3551         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3552
3553         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3554             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3555             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3556             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3557         } elsif ($guest_agent->{type} eq 'isa') {
3558             push @$devices, '-device', "isa-serial,chardev=qga0";
3559         }
3560     }
3561
3562     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3563     if ($rng && $version_guard->(4, 1, 2)) {
3564         check_rng_source($rng->{source});
3565
3566         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3567         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3568         my $limiter_str = "";
3569         if ($max_bytes) {
3570             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3571         }
3572
3573         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3574         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3575         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3576     }
3577
3578     my $spice_port;
3579
3580     if ($qxlnum) {
3581         if ($qxlnum > 1) {
3582             if ($winversion){
3583                 for (my $i = 1; $i < $qxlnum; $i++){
3584                     push @$devices, '-device', print_vga_device(
3585                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3586                 }
3587             } else {
3588                 # assume other OS works like Linux
3589                 my ($ram, $vram) = ("134217728", "67108864");
3590                 if ($vga->{memory}) {
3591                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3592                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3593                 }
3594                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3595                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3596             }
3597         }
3598
3599         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3600
3601         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3602         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3603         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3604
3605         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3606         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3607         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3608
3609         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3610         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3611
3612         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3613         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3614         if ($spice_enhancement->{foldersharing}) {
3615             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3616             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3617         }
3618
3619         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3620         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3621             if $spice_enhancement->{videostreaming};
3622
3623         push @$devices, '-spice', "$spice_opts";
3624     }
3625
3626     # enable balloon by default, unless explicitly disabled
3627     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3628         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3629         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3630     }
3631
3632     if ($conf->{watchdog}) {
3633         my $wdopts = parse_watchdog($conf->{watchdog});
3634         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3635         my $watchdog = $wdopts->{model} || 'i6300esb';
3636         push @$devices, '-device', "$watchdog$pciaddr";
3637         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3638     }
3639
3640     my $vollist = [];
3641     my $scsicontroller = {};
3642     my $ahcicontroller = {};
3643     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3644
3645     # Add iscsi initiator name if available
3646     if (my $initiator = get_initiator_name()) {
3647         push @$devices, '-iscsi', "initiator-name=$initiator";
3648     }
3649
3650     PVE::QemuConfig->foreach_volume($conf, sub {
3651         my ($ds, $drive) = @_;
3652
3653         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3654             check_volume_storage_type($storecfg, $drive->{file});
3655             push @$vollist, $drive->{file};
3656         }
3657
3658         # ignore efidisk here, already added in bios/fw handling code above
3659         return if $drive->{interface} eq 'efidisk';
3660
3661         $use_virtio = 1 if $ds =~ m/^virtio/;
3662
3663         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3664
3665         if ($drive->{interface} eq 'virtio'){
3666            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3667         }
3668
3669         if ($drive->{interface} eq 'scsi') {
3670
3671             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3672
3673             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3674                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3675
3676             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3677             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3678
3679             my $iothread = '';
3680             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3681                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3682                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3683             } elsif ($drive->{iothread}) {
3684                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3685             }
3686
3687             my $queues = '';
3688             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3689                 $queues = ",num_queues=$drive->{queues}";
3690             }
3691
3692             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3693                 if !$scsicontroller->{$controller};
3694             $scsicontroller->{$controller}=1;
3695         }
3696
3697         if ($drive->{interface} eq 'sata') {
3698             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3699             $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3700             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3701                 if !$ahcicontroller->{$controller};
3702             $ahcicontroller->{$controller}=1;
3703         }
3704
3705         my $pbs_conf = $pbs_backing->{$ds};
3706         my $pbs_name = undef;
3707         if ($pbs_conf) {
3708             $pbs_name = "drive-$ds-pbs";
3709             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
3710         }
3711
3712         my $drive_cmd = print_drive_commandline_full(
3713             $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
3714
3715         # extra protection for templates, but SATA and IDE don't support it..
3716         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
3717
3718         push @$devices, '-drive',$drive_cmd;
3719         push @$devices, '-device', print_drivedevice_full(
3720             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3721     });
3722
3723     for (my $i = 0; $i < $MAX_NETS; $i++) {
3724         my $netname = "net$i";
3725
3726         next if !$conf->{$netname};
3727         my $d = parse_net($conf->{$netname});
3728         next if !$d;
3729
3730         $use_virtio = 1 if $d->{model} eq 'virtio';
3731
3732         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
3733
3734         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
3735         push @$devices, '-netdev', $netdevfull;
3736
3737         my $netdevicefull = print_netdevice_full(
3738             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
3739
3740         push @$devices, '-device', $netdevicefull;
3741     }
3742
3743     if ($conf->{ivshmem}) {
3744         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3745
3746         my $bus;
3747         if ($q35) {
3748             $bus = print_pcie_addr("ivshmem");
3749         } else {
3750             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3751         }
3752
3753         my $ivshmem_name = $ivshmem->{name} // $vmid;
3754         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3755
3756         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3757         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
3758             .",size=$ivshmem->{size}M";
3759     }
3760
3761     # pci.4 is nested in pci.1
3762     $bridges->{1} = 1 if $bridges->{4};
3763
3764     if (!$q35) {
3765         # add pci bridges
3766         if (min_version($machine_version, 2, 3)) {
3767            $bridges->{1} = 1;
3768            $bridges->{2} = 1;
3769         }
3770
3771         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3772
3773     }
3774
3775     for my $k (sort {$b cmp $a} keys %$bridges) {
3776         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3777
3778         my $k_name = $k;
3779         if ($k == 2 && $legacy_igd) {
3780             $k_name = "$k-igd";
3781         }
3782         $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
3783
3784         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3785         if ($q35) {
3786             # add after -readconfig pve-q35.cfg
3787             splice @$devices, 2, 0, '-device', $devstr;
3788         } else {
3789             unshift @$devices, '-device', $devstr if $k > 0;
3790         }
3791     }
3792
3793     if (!$kvm) {
3794         push @$machineFlags, 'accel=tcg';
3795     }
3796
3797     my $machine_type_min = $machine_type;
3798     if ($add_pve_version) {
3799         $machine_type_min =~ s/\+pve\d+$//;
3800         $machine_type_min .= "+pve$required_pve_version";
3801     }
3802     push @$machineFlags, "type=${machine_type_min}";
3803
3804     push @$cmd, @$devices;
3805     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
3806     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
3807     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
3808
3809     if (my $vmstate = $conf->{vmstate}) {
3810         my $statepath = PVE::Storage::path($storecfg, $vmstate);
3811         push @$vollist, $vmstate;
3812         push @$cmd, '-loadstate', $statepath;
3813         print "activating and using '$vmstate' as vmstate\n";
3814     }
3815
3816     if (PVE::QemuConfig->is_template($conf)) {
3817         # needed to workaround base volumes being read-only
3818         push @$cmd, '-snapshot';
3819     }
3820
3821     # add custom args
3822     if ($conf->{args}) {
3823         my $aa = PVE::Tools::split_args($conf->{args});
3824         push @$cmd, @$aa;
3825     }
3826
3827     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3828 }
3829
3830 sub check_rng_source {
3831     my ($source) = @_;
3832
3833     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3834     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
3835         if ! -e $source;
3836
3837     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
3838     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
3839         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
3840         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
3841         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
3842             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
3843             ." to the host.\n";
3844     }
3845 }
3846
3847 sub spice_port {
3848     my ($vmid) = @_;
3849
3850     my $res = mon_cmd($vmid, 'query-spice');
3851
3852     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3853 }
3854
3855 sub vm_devices_list {
3856     my ($vmid) = @_;
3857
3858     my $res = mon_cmd($vmid, 'query-pci');
3859     my $devices_to_check = [];
3860     my $devices = {};
3861     foreach my $pcibus (@$res) {
3862         push @$devices_to_check, @{$pcibus->{devices}},
3863     }
3864
3865     while (@$devices_to_check) {
3866         my $to_check = [];
3867         for my $d (@$devices_to_check) {
3868             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3869             next if !$d->{'pci_bridge'};
3870
3871             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3872             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3873         }
3874         $devices_to_check = $to_check;
3875     }
3876
3877     my $resblock = mon_cmd($vmid, 'query-block');
3878     foreach my $block (@$resblock) {
3879         if($block->{device} =~ m/^drive-(\S+)/){
3880                 $devices->{$1} = 1;
3881         }
3882     }
3883
3884     my $resmice = mon_cmd($vmid, 'query-mice');
3885     foreach my $mice (@$resmice) {
3886         if ($mice->{name} eq 'QEMU HID Tablet') {
3887             $devices->{tablet} = 1;
3888             last;
3889         }
3890     }
3891
3892     # for usb devices there is no query-usb
3893     # but we can iterate over the entries in
3894     # qom-list path=/machine/peripheral
3895     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3896     foreach my $per (@$resperipheral) {
3897         if ($per->{name} =~ m/^usb\d+$/) {
3898             $devices->{$per->{name}} = 1;
3899         }
3900     }
3901
3902     return $devices;
3903 }
3904
3905 sub vm_deviceplug {
3906     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3907
3908     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3909
3910     my $devices_list = vm_devices_list($vmid);
3911     return 1 if defined($devices_list->{$deviceid});
3912
3913     # add PCI bridge if we need it for the device
3914     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
3915
3916     if ($deviceid eq 'tablet') {
3917
3918         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
3919
3920     } elsif ($deviceid eq 'keyboard') {
3921
3922         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
3923
3924     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3925
3926         die "usb hotplug currently not reliable\n";
3927         # since we can't reliably hot unplug all added usb devices and usb
3928         # passthrough breaks live migration we disable usb hotplugging for now
3929         #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3930
3931     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3932
3933         qemu_iothread_add($vmid, $deviceid, $device);
3934
3935         qemu_driveadd($storecfg, $vmid, $device);
3936         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3937
3938         qemu_deviceadd($vmid, $devicefull);
3939         eval { qemu_deviceaddverify($vmid, $deviceid); };
3940         if (my $err = $@) {
3941             eval { qemu_drivedel($vmid, $deviceid); };
3942             warn $@ if $@;
3943             die $err;
3944         }
3945
3946     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3947
3948
3949         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3950         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3951         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3952
3953         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3954
3955         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3956             qemu_iothread_add($vmid, $deviceid, $device);
3957             $devicefull .= ",iothread=iothread-$deviceid";
3958         }
3959
3960         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3961             $devicefull .= ",num_queues=$device->{queues}";
3962         }
3963
3964         qemu_deviceadd($vmid, $devicefull);
3965         qemu_deviceaddverify($vmid, $deviceid);
3966
3967     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3968
3969         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
3970         qemu_driveadd($storecfg, $vmid, $device);
3971
3972         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3973         eval { qemu_deviceadd($vmid, $devicefull); };
3974         if (my $err = $@) {
3975             eval { qemu_drivedel($vmid, $deviceid); };
3976             warn $@ if $@;
3977             die $err;
3978         }
3979
3980     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3981
3982         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
3983
3984         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
3985         my $use_old_bios_files = undef;
3986         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3987
3988         my $netdevicefull = print_netdevice_full(
3989             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
3990         qemu_deviceadd($vmid, $netdevicefull);
3991         eval {
3992             qemu_deviceaddverify($vmid, $deviceid);
3993             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
3994         };
3995         if (my $err = $@) {
3996             eval { qemu_netdevdel($vmid, $deviceid); };
3997             warn $@ if $@;
3998             die $err;
3999         }
4000
4001     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4002
4003         my $bridgeid = $2;
4004         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4005         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4006
4007         qemu_deviceadd($vmid, $devicefull);
4008         qemu_deviceaddverify($vmid, $deviceid);
4009
4010     } else {
4011         die "can't hotplug device '$deviceid'\n";
4012     }
4013
4014     return 1;
4015 }
4016
4017 # fixme: this should raise exceptions on error!
4018 sub vm_deviceunplug {
4019     my ($vmid, $conf, $deviceid) = @_;
4020
4021     my $devices_list = vm_devices_list($vmid);
4022     return 1 if !defined($devices_list->{$deviceid});
4023
4024     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
4025     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4026
4027     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4028
4029         qemu_devicedel($vmid, $deviceid);
4030
4031     } elsif ($deviceid =~ m/^usb\d+$/) {
4032
4033         die "usb hotplug currently not reliable\n";
4034         # when unplugging usb devices this way, there may be remaining usb
4035         # controllers/hubs so we disable it for now
4036         #qemu_devicedel($vmid, $deviceid);
4037         #qemu_devicedelverify($vmid, $deviceid);
4038
4039     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4040
4041         qemu_devicedel($vmid, $deviceid);
4042         qemu_devicedelverify($vmid, $deviceid);
4043         qemu_drivedel($vmid, $deviceid);
4044         qemu_iothread_del($conf, $vmid, $deviceid);
4045
4046     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4047
4048         qemu_devicedel($vmid, $deviceid);
4049         qemu_devicedelverify($vmid, $deviceid);
4050         qemu_iothread_del($conf, $vmid, $deviceid);
4051
4052     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4053
4054         qemu_devicedel($vmid, $deviceid);
4055         qemu_drivedel($vmid, $deviceid);
4056         qemu_deletescsihw($conf, $vmid, $deviceid);
4057
4058     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4059
4060         qemu_devicedel($vmid, $deviceid);
4061         qemu_devicedelverify($vmid, $deviceid);
4062         qemu_netdevdel($vmid, $deviceid);
4063
4064     } else {
4065         die "can't unplug device '$deviceid'\n";
4066     }
4067
4068     return 1;
4069 }
4070
4071 sub qemu_deviceadd {
4072     my ($vmid, $devicefull) = @_;
4073
4074     $devicefull = "driver=".$devicefull;
4075     my %options =  split(/[=,]/, $devicefull);
4076
4077     mon_cmd($vmid, "device_add" , %options);
4078 }
4079
4080 sub qemu_devicedel {
4081     my ($vmid, $deviceid) = @_;
4082
4083     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4084 }
4085
4086 sub qemu_iothread_add {
4087     my($vmid, $deviceid, $device) = @_;
4088
4089     if ($device->{iothread}) {
4090         my $iothreads = vm_iothreads_list($vmid);
4091         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4092     }
4093 }
4094
4095 sub qemu_iothread_del {
4096     my($conf, $vmid, $deviceid) = @_;
4097
4098     my $confid = $deviceid;
4099     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4100         $confid = 'scsi' . $1;
4101     }
4102     my $device = parse_drive($confid, $conf->{$confid});
4103     if ($device->{iothread}) {
4104         my $iothreads = vm_iothreads_list($vmid);
4105         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4106     }
4107 }
4108
4109 sub qemu_objectadd {
4110     my($vmid, $objectid, $qomtype) = @_;
4111
4112     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4113
4114     return 1;
4115 }
4116
4117 sub qemu_objectdel {
4118     my($vmid, $objectid) = @_;
4119
4120     mon_cmd($vmid, "object-del", id => $objectid);
4121
4122     return 1;
4123 }
4124
4125 sub qemu_driveadd {
4126     my ($storecfg, $vmid, $device) = @_;
4127
4128     my $kvmver = get_running_qemu_version($vmid);
4129     my $io_uring = min_version($kvmver, 6, 0);
4130     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4131     $drive =~ s/\\/\\\\/g;
4132     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4133
4134     # If the command succeeds qemu prints: "OK"
4135     return 1 if $ret =~ m/OK/s;
4136
4137     die "adding drive failed: $ret\n";
4138 }
4139
4140 sub qemu_drivedel {
4141     my($vmid, $deviceid) = @_;
4142
4143     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4144     $ret =~ s/^\s+//;
4145
4146     return 1 if $ret eq "";
4147
4148     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4149     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4150
4151     die "deleting drive $deviceid failed : $ret\n";
4152 }
4153
4154 sub qemu_deviceaddverify {
4155     my ($vmid, $deviceid) = @_;
4156
4157     for (my $i = 0; $i <= 5; $i++) {
4158          my $devices_list = vm_devices_list($vmid);
4159          return 1 if defined($devices_list->{$deviceid});
4160          sleep 1;
4161     }
4162
4163     die "error on hotplug device '$deviceid'\n";
4164 }
4165
4166
4167 sub qemu_devicedelverify {
4168     my ($vmid, $deviceid) = @_;
4169
4170     # need to verify that the device is correctly removed as device_del
4171     # is async and empty return is not reliable
4172
4173     for (my $i = 0; $i <= 5; $i++) {
4174          my $devices_list = vm_devices_list($vmid);
4175          return 1 if !defined($devices_list->{$deviceid});
4176          sleep 1;
4177     }
4178
4179     die "error on hot-unplugging device '$deviceid'\n";
4180 }
4181
4182 sub qemu_findorcreatescsihw {
4183     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4184
4185     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4186
4187     my $scsihwid="$controller_prefix$controller";
4188     my $devices_list = vm_devices_list($vmid);
4189
4190     if(!defined($devices_list->{$scsihwid})) {
4191         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4192     }
4193
4194     return 1;
4195 }
4196
4197 sub qemu_deletescsihw {
4198     my ($conf, $vmid, $opt) = @_;
4199
4200     my $device = parse_drive($opt, $conf->{$opt});
4201
4202     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4203         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4204         return 1;
4205     }
4206
4207     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4208
4209     my $devices_list = vm_devices_list($vmid);
4210     foreach my $opt (keys %{$devices_list}) {
4211         if (is_valid_drivename($opt)) {
4212             my $drive = parse_drive($opt, $conf->{$opt});
4213             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4214                 return 1;
4215             }
4216         }
4217     }
4218
4219     my $scsihwid="scsihw$controller";
4220
4221     vm_deviceunplug($vmid, $conf, $scsihwid);
4222
4223     return 1;
4224 }
4225
4226 sub qemu_add_pci_bridge {
4227     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4228
4229     my $bridges = {};
4230
4231     my $bridgeid;
4232
4233     print_pci_addr($device, $bridges, $arch, $machine_type);
4234
4235     while (my ($k, $v) = each %$bridges) {
4236         $bridgeid = $k;
4237     }
4238     return 1 if !defined($bridgeid) || $bridgeid < 1;
4239
4240     my $bridge = "pci.$bridgeid";
4241     my $devices_list = vm_devices_list($vmid);
4242
4243     if (!defined($devices_list->{$bridge})) {
4244         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4245     }
4246
4247     return 1;
4248 }
4249
4250 sub qemu_set_link_status {
4251     my ($vmid, $device, $up) = @_;
4252
4253     mon_cmd($vmid, "set_link", name => $device,
4254                up => $up ? JSON::true : JSON::false);
4255 }
4256
4257 sub qemu_netdevadd {
4258     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4259
4260     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4261     my %options =  split(/[=,]/, $netdev);
4262
4263     if (defined(my $vhost = $options{vhost})) {
4264         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4265     }
4266
4267     if (defined(my $queues = $options{queues})) {
4268         $options{queues} = $queues + 0;
4269     }
4270
4271     mon_cmd($vmid, "netdev_add",  %options);
4272     return 1;
4273 }
4274
4275 sub qemu_netdevdel {
4276     my ($vmid, $deviceid) = @_;
4277
4278     mon_cmd($vmid, "netdev_del", id => $deviceid);
4279 }
4280
4281 sub qemu_usb_hotplug {
4282     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4283
4284     return if !$device;
4285
4286     # remove the old one first
4287     vm_deviceunplug($vmid, $conf, $deviceid);
4288
4289     # check if xhci controller is necessary and available
4290     if ($device->{usb3}) {
4291
4292         my $devicelist = vm_devices_list($vmid);
4293
4294         if (!$devicelist->{xhci}) {
4295             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4296             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4297         }
4298     }
4299     my $d = parse_usb_device($device->{host});
4300     $d->{usb3} = $device->{usb3};
4301
4302     # add the new one
4303     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4304 }
4305
4306 sub qemu_cpu_hotplug {
4307     my ($vmid, $conf, $vcpus) = @_;
4308
4309     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4310
4311     my $sockets = 1;
4312     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4313     $sockets = $conf->{sockets} if  $conf->{sockets};
4314     my $cores = $conf->{cores} || 1;
4315     my $maxcpus = $sockets * $cores;
4316
4317     $vcpus = $maxcpus if !$vcpus;
4318
4319     die "you can't add more vcpus than maxcpus\n"
4320         if $vcpus > $maxcpus;
4321
4322     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4323
4324     if ($vcpus < $currentvcpus) {
4325
4326         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4327
4328             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4329                 qemu_devicedel($vmid, "cpu$i");
4330                 my $retry = 0;
4331                 my $currentrunningvcpus = undef;
4332                 while (1) {
4333                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4334                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4335                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4336                     $retry++;
4337                     sleep 1;
4338                 }
4339                 #update conf after each succesfull cpu unplug
4340                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4341                 PVE::QemuConfig->write_config($vmid, $conf);
4342             }
4343         } else {
4344             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4345         }
4346
4347         return;
4348     }
4349
4350     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4351     die "vcpus in running vm does not match its configuration\n"
4352         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4353
4354     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4355
4356         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4357             my $cpustr = print_cpu_device($conf, $i);
4358             qemu_deviceadd($vmid, $cpustr);
4359
4360             my $retry = 0;
4361             my $currentrunningvcpus = undef;
4362             while (1) {
4363                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4364                 last if scalar(@{$currentrunningvcpus}) == $i;
4365                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4366                 sleep 1;
4367                 $retry++;
4368             }
4369             #update conf after each succesfull cpu hotplug
4370             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4371             PVE::QemuConfig->write_config($vmid, $conf);
4372         }
4373     } else {
4374
4375         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4376             mon_cmd($vmid, "cpu-add", id => int($i));
4377         }
4378     }
4379 }
4380
4381 sub qemu_block_set_io_throttle {
4382     my ($vmid, $deviceid,
4383         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4384         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4385         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4386         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4387
4388     return if !check_running($vmid) ;
4389
4390     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4391         bps => int($bps),
4392         bps_rd => int($bps_rd),
4393         bps_wr => int($bps_wr),
4394         iops => int($iops),
4395         iops_rd => int($iops_rd),
4396         iops_wr => int($iops_wr),
4397         bps_max => int($bps_max),
4398         bps_rd_max => int($bps_rd_max),
4399         bps_wr_max => int($bps_wr_max),
4400         iops_max => int($iops_max),
4401         iops_rd_max => int($iops_rd_max),
4402         iops_wr_max => int($iops_wr_max),
4403         bps_max_length => int($bps_max_length),
4404         bps_rd_max_length => int($bps_rd_max_length),
4405         bps_wr_max_length => int($bps_wr_max_length),
4406         iops_max_length => int($iops_max_length),
4407         iops_rd_max_length => int($iops_rd_max_length),
4408         iops_wr_max_length => int($iops_wr_max_length),
4409     );
4410
4411 }
4412
4413 sub qemu_block_resize {
4414     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4415
4416     my $running = check_running($vmid);
4417
4418     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4419
4420     return if !$running;
4421
4422     my $padding = (1024 - $size % 1024) % 1024;
4423     $size = $size + $padding;
4424
4425     mon_cmd(
4426         $vmid,
4427         "block_resize",
4428         device => $deviceid,
4429         size => int($size),
4430         timeout => 60,
4431     );
4432 }
4433
4434 sub qemu_volume_snapshot {
4435     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4436
4437     my $running = check_running($vmid);
4438
4439     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4440         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4441     } else {
4442         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4443     }
4444 }
4445
4446 sub qemu_volume_snapshot_delete {
4447     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4448
4449     my $running = check_running($vmid);
4450
4451     if($running) {
4452
4453         $running = undef;
4454         my $conf = PVE::QemuConfig->load_config($vmid);
4455         PVE::QemuConfig->foreach_volume($conf, sub {
4456             my ($ds, $drive) = @_;
4457             $running = 1 if $drive->{file} eq $volid;
4458         });
4459     }
4460
4461     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4462         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4463     } else {
4464         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4465     }
4466 }
4467
4468 sub set_migration_caps {
4469     my ($vmid, $savevm) = @_;
4470
4471     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4472
4473     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4474     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4475
4476     my $cap_ref = [];
4477
4478     my $enabled_cap = {
4479         "auto-converge" => 1,
4480         "xbzrle" => 1,
4481         "x-rdma-pin-all" => 0,
4482         "zero-blocks" => 0,
4483         "compress" => 0,
4484         "dirty-bitmaps" => $dirty_bitmaps,
4485     };
4486
4487     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4488
4489     for my $supported_capability (@$supported_capabilities) {
4490         push @$cap_ref, {
4491             capability => $supported_capability->{capability},
4492             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4493         };
4494     }
4495
4496     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4497 }
4498
4499 sub foreach_volid {
4500     my ($conf, $func, @param) = @_;
4501
4502     my $volhash = {};
4503
4504     my $test_volid = sub {
4505         my ($key, $drive, $snapname) = @_;
4506
4507         my $volid = $drive->{file};
4508         return if !$volid;
4509
4510         $volhash->{$volid}->{cdrom} //= 1;
4511         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4512
4513         my $replicate = $drive->{replicate} // 1;
4514         $volhash->{$volid}->{replicate} //= 0;
4515         $volhash->{$volid}->{replicate} = 1 if $replicate;
4516
4517         $volhash->{$volid}->{shared} //= 0;
4518         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4519
4520         $volhash->{$volid}->{referenced_in_config} //= 0;
4521         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4522
4523         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4524             if defined($snapname);
4525
4526         my $size = $drive->{size};
4527         $volhash->{$volid}->{size} //= $size if $size;
4528
4529         $volhash->{$volid}->{is_vmstate} //= 0;
4530         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4531
4532         $volhash->{$volid}->{is_unused} //= 0;
4533         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4534
4535         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4536     };
4537
4538     my $include_opts = {
4539         extra_keys => ['vmstate'],
4540         include_unused => 1,
4541     };
4542
4543     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4544     foreach my $snapname (keys %{$conf->{snapshots}}) {
4545         my $snap = $conf->{snapshots}->{$snapname};
4546         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4547     }
4548
4549     foreach my $volid (keys %$volhash) {
4550         &$func($volid, $volhash->{$volid}, @param);
4551     }
4552 }
4553
4554 my $fast_plug_option = {
4555     'lock' => 1,
4556     'name' => 1,
4557     'onboot' => 1,
4558     'shares' => 1,
4559     'startup' => 1,
4560     'description' => 1,
4561     'protection' => 1,
4562     'vmstatestorage' => 1,
4563     'hookscript' => 1,
4564     'tags' => 1,
4565 };
4566
4567 # hotplug changes in [PENDING]
4568 # $selection hash can be used to only apply specified options, for
4569 # example: { cores => 1 } (only apply changed 'cores')
4570 # $errors ref is used to return error messages
4571 sub vmconfig_hotplug_pending {
4572     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4573
4574     my $defaults = load_defaults();
4575     my $arch = get_vm_arch($conf);
4576     my $machine_type = get_vm_machine($conf, undef, $arch);
4577
4578     # commit values which do not have any impact on running VM first
4579     # Note: those option cannot raise errors, we we do not care about
4580     # $selection and always apply them.
4581
4582     my $add_error = sub {
4583         my ($opt, $msg) = @_;
4584         $errors->{$opt} = "hotplug problem - $msg";
4585     };
4586
4587     my $changes = 0;
4588     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4589         if ($fast_plug_option->{$opt}) {
4590             $conf->{$opt} = $conf->{pending}->{$opt};
4591             delete $conf->{pending}->{$opt};
4592             $changes = 1;
4593         }
4594     }
4595
4596     if ($changes) {
4597         PVE::QemuConfig->write_config($vmid, $conf);
4598     }
4599
4600     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4601
4602     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4603     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4604     foreach my $opt (sort keys %$pending_delete_hash) {
4605         next if $selection && !$selection->{$opt};
4606         my $force = $pending_delete_hash->{$opt}->{force};
4607         eval {
4608             if ($opt eq 'hotplug') {
4609                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4610             } elsif ($opt eq 'tablet') {
4611                 die "skip\n" if !$hotplug_features->{usb};
4612                 if ($defaults->{tablet}) {
4613                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4614                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4615                         if $arch eq 'aarch64';
4616                 } else {
4617                     vm_deviceunplug($vmid, $conf, 'tablet');
4618                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4619                 }
4620             } elsif ($opt =~ m/^usb\d+/) {
4621                 die "skip\n";
4622                 # since we cannot reliably hot unplug usb devices we are disabling it
4623                 #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4624                 #vm_deviceunplug($vmid, $conf, $opt);
4625             } elsif ($opt eq 'vcpus') {
4626                 die "skip\n" if !$hotplug_features->{cpu};
4627                 qemu_cpu_hotplug($vmid, $conf, undef);
4628             } elsif ($opt eq 'balloon') {
4629                 # enable balloon device is not hotpluggable
4630                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4631                 # here we reset the ballooning value to memory
4632                 my $balloon = $conf->{memory} || $defaults->{memory};
4633                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4634             } elsif ($fast_plug_option->{$opt}) {
4635                 # do nothing
4636             } elsif ($opt =~ m/^net(\d+)$/) {
4637                 die "skip\n" if !$hotplug_features->{network};
4638                 vm_deviceunplug($vmid, $conf, $opt);
4639             } elsif (is_valid_drivename($opt)) {
4640                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4641                 vm_deviceunplug($vmid, $conf, $opt);
4642                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4643             } elsif ($opt =~ m/^memory$/) {
4644                 die "skip\n" if !$hotplug_features->{memory};
4645                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4646             } elsif ($opt eq 'cpuunits') {
4647                 $cgroup->change_cpu_shares(undef, 1024);
4648             } elsif ($opt eq 'cpulimit') {
4649                 $cgroup->change_cpu_quota(-1, 100000);
4650             } else {
4651                 die "skip\n";
4652             }
4653         };
4654         if (my $err = $@) {
4655             &$add_error($opt, $err) if $err ne "skip\n";
4656         } else {
4657             delete $conf->{$opt};
4658             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4659         }
4660     }
4661
4662     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4663     $apply_pending_cloudinit = sub {
4664         return if $apply_pending_cloudinit_done; # once is enough
4665         $apply_pending_cloudinit_done = 1; # once is enough
4666
4667         my ($key, $value) = @_;
4668
4669         my @cloudinit_opts = keys %$confdesc_cloudinit;
4670         foreach my $opt (keys %{$conf->{pending}}) {
4671             next if !grep { $_ eq $opt } @cloudinit_opts;
4672             $conf->{$opt} = delete $conf->{pending}->{$opt};
4673         }
4674
4675         my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4676         foreach my $opt (sort keys %$pending_delete_hash) {
4677             next if !grep { $_ eq $opt } @cloudinit_opts;
4678             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4679             delete $conf->{$opt};
4680         }
4681
4682         my $new_conf = { %$conf };
4683         $new_conf->{$key} = $value;
4684         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4685     };
4686
4687     foreach my $opt (keys %{$conf->{pending}}) {
4688         next if $selection && !$selection->{$opt};
4689         my $value = $conf->{pending}->{$opt};
4690         eval {
4691             if ($opt eq 'hotplug') {
4692                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4693             } elsif ($opt eq 'tablet') {
4694                 die "skip\n" if !$hotplug_features->{usb};
4695                 if ($value == 1) {
4696                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4697                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4698                         if $arch eq 'aarch64';
4699                 } elsif ($value == 0) {
4700                     vm_deviceunplug($vmid, $conf, 'tablet');
4701                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4702                 }
4703             } elsif ($opt =~ m/^usb\d+$/) {
4704                 die "skip\n";
4705                 # since we cannot reliably hot unplug usb devices we disable it for now
4706                 #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4707                 #my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4708                 #die "skip\n" if !$d;
4709                 #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4710             } elsif ($opt eq 'vcpus') {
4711                 die "skip\n" if !$hotplug_features->{cpu};
4712                 qemu_cpu_hotplug($vmid, $conf, $value);
4713             } elsif ($opt eq 'balloon') {
4714                 # enable/disable balloning device is not hotpluggable
4715                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4716                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4717                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4718
4719                 # allow manual ballooning if shares is set to zero
4720                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4721                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4722                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4723                 }
4724             } elsif ($opt =~ m/^net(\d+)$/) {
4725                 # some changes can be done without hotplug
4726                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4727                                     $vmid, $opt, $value, $arch, $machine_type);
4728             } elsif (is_valid_drivename($opt)) {
4729                 die "skip\n" if $opt eq 'efidisk0';
4730                 # some changes can be done without hotplug
4731                 my $drive = parse_drive($opt, $value);
4732                 if (drive_is_cloudinit($drive)) {
4733                     &$apply_pending_cloudinit($opt, $value);
4734                 }
4735                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4736                                      $vmid, $opt, $value, $arch, $machine_type);
4737             } elsif ($opt =~ m/^memory$/) { #dimms
4738                 die "skip\n" if !$hotplug_features->{memory};
4739                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4740             } elsif ($opt eq 'cpuunits') {
4741                 $cgroup->change_cpu_shares($conf->{pending}->{$opt}, 1024);
4742             } elsif ($opt eq 'cpulimit') {
4743                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4744                 $cgroup->change_cpu_quota($cpulimit, 100000);
4745             } else {
4746                 die "skip\n";  # skip non-hot-pluggable options
4747             }
4748         };
4749         if (my $err = $@) {
4750             &$add_error($opt, $err) if $err ne "skip\n";
4751         } else {
4752             $conf->{$opt} = $value;
4753             delete $conf->{pending}->{$opt};
4754         }
4755     }
4756
4757     PVE::QemuConfig->write_config($vmid, $conf);
4758 }
4759
4760 sub try_deallocate_drive {
4761     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4762
4763     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4764         my $volid = $drive->{file};
4765         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4766             my $sid = PVE::Storage::parse_volume_id($volid);
4767             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4768
4769             # check if the disk is really unused
4770             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4771                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4772             PVE::Storage::vdisk_free($storecfg, $volid);
4773             return 1;
4774         } else {
4775             # If vm is not owner of this disk remove from config
4776             return 1;
4777         }
4778     }
4779
4780     return;
4781 }
4782
4783 sub vmconfig_delete_or_detach_drive {
4784     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4785
4786     my $drive = parse_drive($opt, $conf->{$opt});
4787
4788     my $rpcenv = PVE::RPCEnvironment::get();
4789     my $authuser = $rpcenv->get_user();
4790
4791     if ($force) {
4792         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4793         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4794     } else {
4795         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4796     }
4797 }
4798
4799
4800
4801 sub vmconfig_apply_pending {
4802     my ($vmid, $conf, $storecfg, $errors) = @_;
4803
4804     my $add_apply_error = sub {
4805         my ($opt, $msg) = @_;
4806         my $err_msg = "unable to apply pending change $opt : $msg";
4807         $errors->{$opt} = $err_msg;
4808         warn $err_msg;
4809     };
4810
4811     # cold plug
4812
4813     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4814     foreach my $opt (sort keys %$pending_delete_hash) {
4815         my $force = $pending_delete_hash->{$opt}->{force};
4816         eval {
4817             if ($opt =~ m/^unused/) {
4818                 die "internal error";
4819             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4820                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4821             }
4822         };
4823         if (my $err = $@) {
4824             $add_apply_error->($opt, $err);
4825         } else {
4826             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4827             delete $conf->{$opt};
4828         }
4829     }
4830
4831     PVE::QemuConfig->cleanup_pending($conf);
4832
4833     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4834         next if $opt eq 'delete'; # just to be sure
4835         eval {
4836             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4837                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4838             }
4839         };
4840         if (my $err = $@) {
4841             $add_apply_error->($opt, $err);
4842         } else {
4843             $conf->{$opt} = delete $conf->{pending}->{$opt};
4844         }
4845     }
4846
4847     # write all changes at once to avoid unnecessary i/o
4848     PVE::QemuConfig->write_config($vmid, $conf);
4849 }
4850
4851 sub vmconfig_update_net {
4852     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4853
4854     my $newnet = parse_net($value);
4855
4856     if ($conf->{$opt}) {
4857         my $oldnet = parse_net($conf->{$opt});
4858
4859         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4860             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4861             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4862             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4863
4864             # for non online change, we try to hot-unplug
4865             die "skip\n" if !$hotplug;
4866             vm_deviceunplug($vmid, $conf, $opt);
4867         } else {
4868
4869             die "internal error" if $opt !~ m/net(\d+)/;
4870             my $iface = "tap${vmid}i$1";
4871
4872             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4873                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4874                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4875                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4876                 PVE::Network::tap_unplug($iface);
4877
4878                 if ($have_sdn) {
4879                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4880                 } else {
4881                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4882                 }
4883             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4884                 # Rate can be applied on its own but any change above needs to
4885                 # include the rate in tap_plug since OVS resets everything.
4886                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4887             }
4888
4889             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4890                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4891             }
4892
4893             return 1;
4894         }
4895     }
4896
4897     if ($hotplug) {
4898         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4899     } else {
4900         die "skip\n";
4901     }
4902 }
4903
4904 sub vmconfig_update_disk {
4905     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4906
4907     my $drive = parse_drive($opt, $value);
4908
4909     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
4910         my $media = $drive->{media} || 'disk';
4911         my $oldmedia = $old_drive->{media} || 'disk';
4912         die "unable to change media type\n" if $media ne $oldmedia;
4913
4914         if (!drive_is_cdrom($old_drive)) {
4915
4916             if ($drive->{file} ne $old_drive->{file}) {
4917
4918                 die "skip\n" if !$hotplug;
4919
4920                 # unplug and register as unused
4921                 vm_deviceunplug($vmid, $conf, $opt);
4922                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4923
4924             } else {
4925                 # update existing disk
4926
4927                 # skip non hotpluggable value
4928                 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4929                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4930                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4931                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
4932                     safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
4933                     die "skip\n";
4934                 }
4935
4936                 # apply throttle
4937                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4938                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4939                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4940                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4941                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4942                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4943                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4944                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4945                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4946                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4947                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4948                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4949                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4950                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4951                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4952                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4953                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4954                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4955
4956                     qemu_block_set_io_throttle(
4957                         $vmid,"drive-$opt",
4958                         ($drive->{mbps} || 0)*1024*1024,
4959                         ($drive->{mbps_rd} || 0)*1024*1024,
4960                         ($drive->{mbps_wr} || 0)*1024*1024,
4961                          $drive->{iops} || 0,
4962                          $drive->{iops_rd} || 0,
4963                          $drive->{iops_wr} || 0,
4964                         ($drive->{mbps_max} || 0)*1024*1024,
4965                         ($drive->{mbps_rd_max} || 0)*1024*1024,
4966                         ($drive->{mbps_wr_max} || 0)*1024*1024,
4967                          $drive->{iops_max} || 0,
4968                          $drive->{iops_rd_max} || 0,
4969                          $drive->{iops_wr_max} || 0,
4970                          $drive->{bps_max_length} || 1,
4971                          $drive->{bps_rd_max_length} || 1,
4972                          $drive->{bps_wr_max_length} || 1,
4973                          $drive->{iops_max_length} || 1,
4974                          $drive->{iops_rd_max_length} || 1,
4975                          $drive->{iops_wr_max_length} || 1,
4976                     );
4977
4978                 }
4979
4980                 return 1;
4981             }
4982
4983         } else { # cdrom
4984
4985             if ($drive->{file} eq 'none') {
4986                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4987                 if (drive_is_cloudinit($old_drive)) {
4988                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4989                 }
4990             } else {
4991                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4992
4993                 # force eject if locked
4994                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4995
4996                 if ($path) {
4997                     mon_cmd($vmid, "blockdev-change-medium",
4998                         id => "$opt", filename => "$path");
4999                 }
5000             }
5001
5002             return 1;
5003         }
5004     }
5005
5006     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5007     # hotplug new disks
5008     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5009     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5010 }
5011
5012 # called in locked context by incoming migration
5013 sub vm_migrate_get_nbd_disks {
5014     my ($storecfg, $conf, $replicated_volumes) = @_;
5015
5016     my $local_volumes = {};
5017     PVE::QemuConfig->foreach_volume($conf, sub {
5018         my ($ds, $drive) = @_;
5019
5020         return if drive_is_cdrom($drive);
5021
5022         my $volid = $drive->{file};
5023
5024         return if !$volid;
5025
5026         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5027
5028         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5029         return if $scfg->{shared};
5030
5031         # replicated disks re-use existing state via bitmap
5032         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5033         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
5034     });
5035     return $local_volumes;
5036 }
5037
5038 # called in locked context by incoming migration
5039 sub vm_migrate_alloc_nbd_disks {
5040     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5041
5042     my $format = undef;
5043
5044     my $nbd = {};
5045     foreach my $opt (sort keys %$source_volumes) {
5046         my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
5047
5048         if ($use_existing) {
5049             $nbd->{$opt}->{drivestr} = print_drive($drive);
5050             $nbd->{$opt}->{volid} = $volid;
5051             $nbd->{$opt}->{replicated} = 1;
5052             next;
5053         }
5054
5055         # If a remote storage is specified and the format of the original
5056         # volume is not available there, fall back to the default format.
5057         # Otherwise use the same format as the original.
5058         if (!$storagemap->{identity}) {
5059             $storeid = map_storage($storagemap, $storeid);
5060             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5061             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5062             my $fileFormat = qemu_img_format($scfg, $volname);
5063             $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
5064         } else {
5065             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5066             $format = qemu_img_format($scfg, $volname);
5067         }
5068
5069         my $size = $drive->{size} / 1024;
5070         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5071         my $newdrive = $drive;
5072         $newdrive->{format} = $format;
5073         $newdrive->{file} = $newvolid;
5074         my $drivestr = print_drive($newdrive);
5075         $nbd->{$opt}->{drivestr} = $drivestr;
5076         $nbd->{$opt}->{volid} = $newvolid;
5077     }
5078
5079     return $nbd;
5080 }
5081
5082 # see vm_start_nolock for parameters, additionally:
5083 # migrate_opts:
5084 #   storagemap = parsed storage map for allocating NBD disks
5085 sub vm_start {
5086     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5087
5088     return PVE::QemuConfig->lock_config($vmid, sub {
5089         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5090
5091         die "you can't start a vm if it's a template\n"
5092             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5093
5094         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5095         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5096
5097         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5098
5099         if ($has_backup_lock && $running) {
5100             # a backup is currently running, attempt to start the guest in the
5101             # existing QEMU instance
5102             return vm_resume($vmid);
5103         }
5104
5105         PVE::QemuConfig->check_lock($conf)
5106             if !($params->{skiplock} || $has_suspended_lock);
5107
5108         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5109
5110         die "VM $vmid already running\n" if $running;
5111
5112         if (my $storagemap = $migrate_opts->{storagemap}) {
5113             my $replicated = $migrate_opts->{replicated_volumes};
5114             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5115             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5116
5117             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5118                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5119             }
5120         }
5121
5122         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5123     });
5124 }
5125
5126
5127 # params:
5128 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5129 #   skiplock => 0/1, skip checking for config lock
5130 #   skiptemplate => 0/1, skip checking whether VM is template
5131 #   forcemachine => to force Qemu machine (rollback/migration)
5132 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5133 #   timeout => in seconds
5134 #   paused => start VM in paused state (backup)
5135 #   resume => resume from hibernation
5136 #   pbs-backing => {
5137 #      sata0 => {
5138 #         repository
5139 #         snapshot
5140 #         keyfile
5141 #         archive
5142 #      },
5143 #      virtio2 => ...
5144 #   }
5145 # migrate_opts:
5146 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5147 #   migratedfrom => source node
5148 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5149 #   network => CIDR of migration network
5150 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5151 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5152 #   replicated_volumes = which volids should be re-used with bitmaps for nbd migration
5153 sub vm_start_nolock {
5154     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5155
5156     my $statefile = $params->{statefile};
5157     my $resume = $params->{resume};
5158
5159     my $migratedfrom = $migrate_opts->{migratedfrom};
5160     my $migration_type = $migrate_opts->{type};
5161
5162     my $res = {};
5163
5164     # clean up leftover reboot request files
5165     eval { clear_reboot_request($vmid); };
5166     warn $@ if $@;
5167
5168     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5169         vmconfig_apply_pending($vmid, $conf, $storecfg);
5170         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5171     }
5172
5173     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5174
5175     my $defaults = load_defaults();
5176
5177     # set environment variable useful inside network script
5178     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5179
5180     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5181
5182     my $forcemachine = $params->{forcemachine};
5183     my $forcecpu = $params->{forcecpu};
5184     if ($resume) {
5185         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5186         $forcemachine = $conf->{runningmachine};
5187         $forcecpu = $conf->{runningcpu};
5188         print "Resuming suspended VM\n";
5189     }
5190
5191     my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid,
5192         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5193
5194     my $migration_ip;
5195     my $get_migration_ip = sub {
5196         my ($nodename) = @_;
5197
5198         return $migration_ip if defined($migration_ip);
5199
5200         my $cidr = $migrate_opts->{network};
5201
5202         if (!defined($cidr)) {
5203             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5204             $cidr = $dc_conf->{migration}->{network};
5205         }
5206
5207         if (defined($cidr)) {
5208             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5209
5210             die "could not get IP: no address configured on local " .
5211                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5212
5213             die "could not get IP: multiple addresses configured on local " .
5214                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5215
5216             $migration_ip = @$ips[0];
5217         }
5218
5219         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5220             if !defined($migration_ip);
5221
5222         return $migration_ip;
5223     };
5224
5225     my $migrate_uri;
5226     if ($statefile) {
5227         if ($statefile eq 'tcp') {
5228             my $localip = "localhost";
5229             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5230             my $nodename = nodename();
5231
5232             if (!defined($migration_type)) {
5233                 if (defined($datacenterconf->{migration}->{type})) {
5234                     $migration_type = $datacenterconf->{migration}->{type};
5235                 } else {
5236                     $migration_type = 'secure';
5237                 }
5238             }
5239
5240             if ($migration_type eq 'insecure') {
5241                 $localip = $get_migration_ip->($nodename);
5242                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5243             }
5244
5245             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5246             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5247             $migrate_uri = "tcp:${localip}:${migrate_port}";
5248             push @$cmd, '-incoming', $migrate_uri;
5249             push @$cmd, '-S';
5250
5251         } elsif ($statefile eq 'unix') {
5252             # should be default for secure migrations as a ssh TCP forward
5253             # tunnel is not deterministic reliable ready and fails regurarly
5254             # to set up in time, so use UNIX socket forwards
5255             my $socket_addr = "/run/qemu-server/$vmid.migrate";
5256             unlink $socket_addr;
5257
5258             $migrate_uri = "unix:$socket_addr";
5259
5260             push @$cmd, '-incoming', $migrate_uri;
5261             push @$cmd, '-S';
5262
5263         } elsif (-e $statefile) {
5264             push @$cmd, '-loadstate', $statefile;
5265         } else {
5266             my $statepath = PVE::Storage::path($storecfg, $statefile);
5267             push @$vollist, $statefile;
5268             push @$cmd, '-loadstate', $statepath;
5269         }
5270     } elsif ($params->{paused}) {
5271         push @$cmd, '-S';
5272     }
5273
5274     # host pci devices
5275     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5276       my $d = parse_hostpci($conf->{"hostpci$i"});
5277       next if !$d;
5278       my $pcidevices = $d->{pciid};
5279       foreach my $pcidevice (@$pcidevices) {
5280             my $pciid = $pcidevice->{id};
5281
5282             my $info = PVE::SysFSTools::pci_device_info("$pciid");
5283             die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5284             die "no pci device info for device '$pciid'\n" if !$info;
5285
5286             if ($d->{mdev}) {
5287                 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5288                 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5289             } else {
5290                 die "can't unbind/bind PCI group to VFIO '$pciid'\n"
5291                     if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5292                 die "can't reset PCI device '$pciid'\n"
5293                     if $info->{has_fl_reset} && !PVE::SysFSTools::pci_dev_reset($info);
5294             }
5295       }
5296     }
5297
5298     PVE::Storage::activate_volumes($storecfg, $vollist);
5299
5300     eval {
5301         run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5302             outfunc => sub {}, errfunc => sub {});
5303     };
5304     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5305     # timeout should be more than enough here...
5306     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5307
5308     my $cpuunits = get_cpuunits($conf);
5309
5310     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5311     my %run_params = (
5312         timeout => $statefile ? undef : $start_timeout,
5313         umask => 0077,
5314         noerr => 1,
5315     );
5316
5317     # when migrating, prefix QEMU output so other side can pick up any
5318     # errors that might occur and show the user
5319     if ($migratedfrom) {
5320         $run_params{quiet} = 1;
5321         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5322     }
5323
5324     my %properties = (
5325         Slice => 'qemu.slice',
5326         KillMode => 'process',
5327         SendSIGKILL => 0,
5328         TimeoutStopUSec => ULONG_MAX, # infinity
5329     );
5330
5331     if (PVE::CGroup::cgroup_mode() == 2) {
5332         $cpuunits = 10000 if $cpuunits >= 10000; # else we get an error
5333         $properties{CPUWeight} = $cpuunits;
5334     } else {
5335         $properties{CPUShares} = $cpuunits;
5336     }
5337
5338     if (my $cpulimit = $conf->{cpulimit}) {
5339         $properties{CPUQuota} = int($cpulimit * 100);
5340     }
5341     $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5342
5343     my $run_qemu = sub {
5344         PVE::Tools::run_fork sub {
5345             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5346
5347             my $exitcode = run_command($cmd, %run_params);
5348             die "QEMU exited with code $exitcode\n" if $exitcode;
5349         };
5350     };
5351
5352     if ($conf->{hugepages}) {
5353
5354         my $code = sub {
5355             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5356             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5357
5358             PVE::QemuServer::Memory::hugepages_mount();
5359             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5360
5361             eval { $run_qemu->() };
5362             if (my $err = $@) {
5363                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5364                     if !$conf->{keephugepages};
5365                 die $err;
5366             }
5367
5368             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5369                 if !$conf->{keephugepages};
5370         };
5371         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5372
5373     } else {
5374         eval { $run_qemu->() };
5375     }
5376
5377     if (my $err = $@) {
5378         # deactivate volumes if start fails
5379         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5380         die "start failed: $err";
5381     }
5382
5383     print "migration listens on $migrate_uri\n" if $migrate_uri;
5384     $res->{migrate_uri} = $migrate_uri;
5385
5386     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5387         eval { mon_cmd($vmid, "cont"); };
5388         warn $@ if $@;
5389     }
5390
5391     #start nbd server for storage migration
5392     if (my $nbd = $migrate_opts->{nbd}) {
5393         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5394
5395         my $migrate_storage_uri;
5396         # nbd_protocol_version > 0 for unix socket support
5397         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5398             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5399             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5400             $migrate_storage_uri = "nbd:unix:$socket_path";
5401         } else {
5402             my $nodename = nodename();
5403             my $localip = $get_migration_ip->($nodename);
5404             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5405             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5406
5407             mon_cmd($vmid, "nbd-server-start", addr => {
5408                 type => 'inet',
5409                 data => {
5410                     host => "${localip}",
5411                     port => "${storage_migrate_port}",
5412                 },
5413             });
5414             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5415             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5416         }
5417
5418         $res->{migrate_storage_uri} = $migrate_storage_uri;
5419
5420         foreach my $opt (sort keys %$nbd) {
5421             my $drivestr = $nbd->{$opt}->{drivestr};
5422             my $volid = $nbd->{$opt}->{volid};
5423             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5424             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5425             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5426             print "re-using replicated volume: $opt - $volid\n"
5427                 if $nbd->{$opt}->{replicated};
5428
5429             $res->{drives}->{$opt} = $nbd->{$opt};
5430             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5431         }
5432     }
5433
5434     if ($migratedfrom) {
5435         eval {
5436             set_migration_caps($vmid);
5437         };
5438         warn $@ if $@;
5439
5440         if ($spice_port) {
5441             print "spice listens on port $spice_port\n";
5442             $res->{spice_port} = $spice_port;
5443             if ($migrate_opts->{spice_ticket}) {
5444                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5445                     $migrate_opts->{spice_ticket});
5446                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5447             }
5448         }
5449
5450     } else {
5451         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5452             if !$statefile && $conf->{balloon};
5453
5454         foreach my $opt (keys %$conf) {
5455             next if $opt !~  m/^net\d+$/;
5456             my $nicconf = parse_net($conf->{$opt});
5457             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5458         }
5459     }
5460
5461     mon_cmd($vmid, 'qom-set',
5462                 path => "machine/peripheral/balloon0",
5463                 property => "guest-stats-polling-interval",
5464                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5465
5466     if ($resume) {
5467         print "Resumed VM, removing state\n";
5468         if (my $vmstate = $conf->{vmstate}) {
5469             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5470             PVE::Storage::vdisk_free($storecfg, $vmstate);
5471         }
5472         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5473         PVE::QemuConfig->write_config($vmid, $conf);
5474     }
5475
5476     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5477
5478     return $res;
5479 }
5480
5481 sub vm_commandline {
5482     my ($storecfg, $vmid, $snapname) = @_;
5483
5484     my $conf = PVE::QemuConfig->load_config($vmid);
5485     my $forcemachine;
5486     my $forcecpu;
5487
5488     if ($snapname) {
5489         my $snapshot = $conf->{snapshots}->{$snapname};
5490         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5491
5492         # check for machine or CPU overrides in snapshot
5493         $forcemachine = $snapshot->{runningmachine};
5494         $forcecpu = $snapshot->{runningcpu};
5495
5496         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5497
5498         $conf = $snapshot;
5499     }
5500
5501     my $defaults = load_defaults();
5502
5503     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults,
5504         $forcemachine, $forcecpu);
5505
5506     return PVE::Tools::cmd2string($cmd);
5507 }
5508
5509 sub vm_reset {
5510     my ($vmid, $skiplock) = @_;
5511
5512     PVE::QemuConfig->lock_config($vmid, sub {
5513
5514         my $conf = PVE::QemuConfig->load_config($vmid);
5515
5516         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5517
5518         mon_cmd($vmid, "system_reset");
5519     });
5520 }
5521
5522 sub get_vm_volumes {
5523     my ($conf) = @_;
5524
5525     my $vollist = [];
5526     foreach_volid($conf, sub {
5527         my ($volid, $attr) = @_;
5528
5529         return if $volid =~ m|^/|;
5530
5531         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5532         return if !$sid;
5533
5534         push @$vollist, $volid;
5535     });
5536
5537     return $vollist;
5538 }
5539
5540 sub vm_stop_cleanup {
5541     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5542
5543     eval {
5544
5545         if (!$keepActive) {
5546             my $vollist = get_vm_volumes($conf);
5547             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5548         }
5549
5550         foreach my $ext (qw(mon qmp pid vnc qga)) {
5551             unlink "/var/run/qemu-server/${vmid}.$ext";
5552         }
5553
5554         if ($conf->{ivshmem}) {
5555             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5556             # just delete it for now, VMs which have this already open do not
5557             # are affected, but new VMs will get a separated one. If this
5558             # becomes an issue we either add some sort of ref-counting or just
5559             # add a "don't delete on stop" flag to the ivshmem format.
5560             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5561         }
5562
5563         foreach my $key (keys %$conf) {
5564             next if $key !~ m/^hostpci(\d+)$/;
5565             my $hostpciindex = $1;
5566             my $d = parse_hostpci($conf->{$key});
5567             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5568
5569             foreach my $pci (@{$d->{pciid}}) {
5570                 my $pciid = $pci->{id};
5571                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5572             }
5573         }
5574
5575         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5576     };
5577     warn $@ if $@; # avoid errors - just warn
5578 }
5579
5580 # call only in locked context
5581 sub _do_vm_stop {
5582     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5583
5584     my $pid = check_running($vmid, $nocheck);
5585     return if !$pid;
5586
5587     my $conf;
5588     if (!$nocheck) {
5589         $conf = PVE::QemuConfig->load_config($vmid);
5590         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5591         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5592             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5593             $timeout = $opts->{down} if $opts->{down};
5594         }
5595         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5596     }
5597
5598     eval {
5599         if ($shutdown) {
5600             if (defined($conf) && get_qga_key($conf, 'enabled')) {
5601                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5602             } else {
5603                 mon_cmd($vmid, "system_powerdown");
5604             }
5605         } else {
5606             mon_cmd($vmid, "quit");
5607         }
5608     };
5609     my $err = $@;
5610
5611     if (!$err) {
5612         $timeout = 60 if !defined($timeout);
5613
5614         my $count = 0;
5615         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5616             $count++;
5617             sleep 1;
5618         }
5619
5620         if ($count >= $timeout) {
5621             if ($force) {
5622                 warn "VM still running - terminating now with SIGTERM\n";
5623                 kill 15, $pid;
5624             } else {
5625                 die "VM quit/powerdown failed - got timeout\n";
5626             }
5627         } else {
5628             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5629             return;
5630         }
5631     } else {
5632        if (!check_running($vmid, $nocheck)) {
5633             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5634             return;
5635        }
5636        if ($force) {
5637             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5638             kill 15, $pid;
5639         } else {
5640             die "VM quit/powerdown failed\n";
5641         }
5642     }
5643
5644     # wait again
5645     $timeout = 10;
5646
5647     my $count = 0;
5648     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5649         $count++;
5650         sleep 1;
5651     }
5652
5653     if ($count >= $timeout) {
5654         warn "VM still running - terminating now with SIGKILL\n";
5655         kill 9, $pid;
5656         sleep 1;
5657     }
5658
5659     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5660 }
5661
5662 # Note: use $nocheck to skip tests if VM configuration file exists.
5663 # We need that when migration VMs to other nodes (files already moved)
5664 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5665 sub vm_stop {
5666     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5667
5668     $force = 1 if !defined($force) && !$shutdown;
5669
5670     if ($migratedfrom){
5671         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5672         kill 15, $pid if $pid;
5673         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5674         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5675         return;
5676     }
5677
5678     PVE::QemuConfig->lock_config($vmid, sub {
5679         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5680    });
5681 }
5682
5683 sub vm_reboot {
5684     my ($vmid, $timeout) = @_;
5685
5686     PVE::QemuConfig->lock_config($vmid, sub {
5687         eval {
5688
5689             # only reboot if running, as qmeventd starts it again on a stop event
5690             return if !check_running($vmid);
5691
5692             create_reboot_request($vmid);
5693
5694             my $storecfg = PVE::Storage::config();
5695             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5696
5697         };
5698         if (my $err = $@) {
5699             # avoid that the next normal shutdown will be confused for a reboot
5700             clear_reboot_request($vmid);
5701             die $err;
5702         }
5703    });
5704 }
5705
5706 # note: if using the statestorage parameter, the caller has to check privileges
5707 sub vm_suspend {
5708     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5709
5710     my $conf;
5711     my $path;
5712     my $storecfg;
5713     my $vmstate;
5714
5715     PVE::QemuConfig->lock_config($vmid, sub {
5716
5717         $conf = PVE::QemuConfig->load_config($vmid);
5718
5719         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5720         PVE::QemuConfig->check_lock($conf)
5721             if !($skiplock || $is_backing_up);
5722
5723         die "cannot suspend to disk during backup\n"
5724             if $is_backing_up && $includestate;
5725
5726         if ($includestate) {
5727             $conf->{lock} = 'suspending';
5728             my $date = strftime("%Y-%m-%d", localtime(time()));
5729             $storecfg = PVE::Storage::config();
5730             if (!$statestorage) {
5731                 $statestorage = find_vmstate_storage($conf, $storecfg);
5732                 # check permissions for the storage
5733                 my $rpcenv = PVE::RPCEnvironment::get();
5734                 if ($rpcenv->{type} ne 'cli') {
5735                     my $authuser = $rpcenv->get_user();
5736                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5737                 }
5738             }
5739
5740
5741             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
5742                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5743             $path = PVE::Storage::path($storecfg, $vmstate);
5744             PVE::QemuConfig->write_config($vmid, $conf);
5745         } else {
5746             mon_cmd($vmid, "stop");
5747         }
5748     });
5749
5750     if ($includestate) {
5751         # save vm state
5752         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5753
5754         eval {
5755             set_migration_caps($vmid, 1);
5756             mon_cmd($vmid, "savevm-start", statefile => $path);
5757             for(;;) {
5758                 my $state = mon_cmd($vmid, "query-savevm");
5759                 if (!$state->{status}) {
5760                     die "savevm not active\n";
5761                 } elsif ($state->{status} eq 'active') {
5762                     sleep(1);
5763                     next;
5764                 } elsif ($state->{status} eq 'completed') {
5765                     print "State saved, quitting\n";
5766                     last;
5767                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5768                     die "query-savevm failed with error '$state->{error}'\n"
5769                 } else {
5770                     die "query-savevm returned status '$state->{status}'\n";
5771                 }
5772             }
5773         };
5774         my $err = $@;
5775
5776         PVE::QemuConfig->lock_config($vmid, sub {
5777             $conf = PVE::QemuConfig->load_config($vmid);
5778             if ($err) {
5779                 # cleanup, but leave suspending lock, to indicate something went wrong
5780                 eval {
5781                     mon_cmd($vmid, "savevm-end");
5782                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5783                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5784                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
5785                     PVE::QemuConfig->write_config($vmid, $conf);
5786                 };
5787                 warn $@ if $@;
5788                 die $err;
5789             }
5790
5791             die "lock changed unexpectedly\n"
5792                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5793
5794             mon_cmd($vmid, "quit");
5795             $conf->{lock} = 'suspended';
5796             PVE::QemuConfig->write_config($vmid, $conf);
5797         });
5798     }
5799 }
5800
5801 sub vm_resume {
5802     my ($vmid, $skiplock, $nocheck) = @_;
5803
5804     PVE::QemuConfig->lock_config($vmid, sub {
5805         my $res = mon_cmd($vmid, 'query-status');
5806         my $resume_cmd = 'cont';
5807         my $reset = 0;
5808
5809         if ($res->{status}) {
5810             return if $res->{status} eq 'running'; # job done, go home
5811             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
5812             $reset = 1 if $res->{status} eq 'shutdown';
5813         }
5814
5815         if (!$nocheck) {
5816
5817             my $conf = PVE::QemuConfig->load_config($vmid);
5818
5819             PVE::QemuConfig->check_lock($conf)
5820                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5821         }
5822
5823         if ($reset) {
5824             # required if a VM shuts down during a backup and we get a resume
5825             # request before the backup finishes for example
5826             mon_cmd($vmid, "system_reset");
5827         }
5828         mon_cmd($vmid, $resume_cmd);
5829     });
5830 }
5831
5832 sub vm_sendkey {
5833     my ($vmid, $skiplock, $key) = @_;
5834
5835     PVE::QemuConfig->lock_config($vmid, sub {
5836
5837         my $conf = PVE::QemuConfig->load_config($vmid);
5838
5839         # there is no qmp command, so we use the human monitor command
5840         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5841         die $res if $res ne '';
5842     });
5843 }
5844
5845 # vzdump restore implementaion
5846
5847 sub tar_archive_read_firstfile {
5848     my $archive = shift;
5849
5850     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5851
5852     # try to detect archive type first
5853     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5854         die "unable to open file '$archive'\n";
5855     my $firstfile = <$fh>;
5856     kill 15, $pid;
5857     close $fh;
5858
5859     die "ERROR: archive contaions no data\n" if !$firstfile;
5860     chomp $firstfile;
5861
5862     return $firstfile;
5863 }
5864
5865 sub tar_restore_cleanup {
5866     my ($storecfg, $statfile) = @_;
5867
5868     print STDERR "starting cleanup\n";
5869
5870     if (my $fd = IO::File->new($statfile, "r")) {
5871         while (defined(my $line = <$fd>)) {
5872             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5873                 my $volid = $2;
5874                 eval {
5875                     if ($volid =~ m|^/|) {
5876                         unlink $volid || die 'unlink failed\n';
5877                     } else {
5878                         PVE::Storage::vdisk_free($storecfg, $volid);
5879                     }
5880                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5881                 };
5882                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5883             } else {
5884                 print STDERR "unable to parse line in statfile - $line";
5885             }
5886         }
5887         $fd->close();
5888     }
5889 }
5890
5891 sub restore_file_archive {
5892     my ($archive, $vmid, $user, $opts) = @_;
5893
5894     return restore_vma_archive($archive, $vmid, $user, $opts)
5895         if $archive eq '-';
5896
5897     my $info = PVE::Storage::archive_info($archive);
5898     my $format = $opts->{format} // $info->{format};
5899     my $comp = $info->{compression};
5900
5901     # try to detect archive format
5902     if ($format eq 'tar') {
5903         return restore_tar_archive($archive, $vmid, $user, $opts);
5904     } else {
5905         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5906     }
5907 }
5908
5909 # hepler to remove disks that will not be used after restore
5910 my $restore_cleanup_oldconf = sub {
5911     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
5912
5913     PVE::QemuConfig->foreach_volume($oldconf, sub {
5914         my ($ds, $drive) = @_;
5915
5916         return if drive_is_cdrom($drive, 1);
5917
5918         my $volid = $drive->{file};
5919         return if !$volid || $volid =~ m|^/|;
5920
5921         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
5922         return if !$path || !$owner || ($owner != $vmid);
5923
5924         # Note: only delete disk we want to restore
5925         # other volumes will become unused
5926         if ($virtdev_hash->{$ds}) {
5927             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
5928             if (my $err = $@) {
5929                 warn $err;
5930             }
5931         }
5932     });
5933
5934     # delete vmstate files, after the restore we have no snapshots anymore
5935     foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5936         my $snap = $oldconf->{snapshots}->{$snapname};
5937         if ($snap->{vmstate}) {
5938             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
5939             if (my $err = $@) {
5940                 warn $err;
5941             }
5942         }
5943     }
5944 };
5945
5946 # Helper to parse vzdump backup device hints
5947 #
5948 # $rpcenv: Environment, used to ckeck storage permissions
5949 # $user: User ID, to check storage permissions
5950 # $storecfg: Storage configuration
5951 # $fh: the file handle for reading the configuration
5952 # $devinfo: should contain device sizes for all backu-up'ed devices
5953 # $options: backup options (pool, default storage)
5954 #
5955 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
5956 my $parse_backup_hints = sub {
5957     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
5958
5959     my $virtdev_hash = {};
5960
5961     while (defined(my $line = <$fh>)) {
5962         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5963             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5964             die "archive does not contain data for drive '$virtdev'\n"
5965                 if !$devinfo->{$devname};
5966
5967             if (defined($options->{storage})) {
5968                 $storeid = $options->{storage} || 'local';
5969             } elsif (!$storeid) {
5970                 $storeid = 'local';
5971             }
5972             $format = 'raw' if !$format;
5973             $devinfo->{$devname}->{devname} = $devname;
5974             $devinfo->{$devname}->{virtdev} = $virtdev;
5975             $devinfo->{$devname}->{format} = $format;
5976             $devinfo->{$devname}->{storeid} = $storeid;
5977
5978             # check permission on storage
5979             my $pool = $options->{pool}; # todo: do we need that?
5980             if ($user ne 'root@pam') {
5981                 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5982             }
5983
5984             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5985         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
5986             my $virtdev = $1;
5987             my $drive = parse_drive($virtdev, $2);
5988             if (drive_is_cloudinit($drive)) {
5989                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5990                 $storeid = $options->{storage} if defined ($options->{storage});
5991                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5992                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
5993
5994                 $virtdev_hash->{$virtdev} = {
5995                     format => $format,
5996                     storeid => $storeid,
5997                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
5998                     is_cloudinit => 1,
5999                 };
6000             }
6001         }
6002     }
6003
6004     return $virtdev_hash;
6005 };
6006
6007 # Helper to allocate and activate all volumes required for a restore
6008 #
6009 # $storecfg: Storage configuration
6010 # $virtdev_hash: as returned by parse_backup_hints()
6011 #
6012 # Returns: { $virtdev => $volid }
6013 my $restore_allocate_devices = sub {
6014     my ($storecfg, $virtdev_hash, $vmid) = @_;
6015
6016     my $map = {};
6017     foreach my $virtdev (sort keys %$virtdev_hash) {
6018         my $d = $virtdev_hash->{$virtdev};
6019         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6020         my $storeid = $d->{storeid};
6021         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6022
6023         # test if requested format is supported
6024         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6025         my $supported = grep { $_ eq $d->{format} } @$validFormats;
6026         $d->{format} = $defFormat if !$supported;
6027
6028         my $name;
6029         if ($d->{is_cloudinit}) {
6030             $name = "vm-$vmid-cloudinit";
6031             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6032             if ($scfg->{path}) {
6033                 $name .= ".$d->{format}";
6034             }
6035         }
6036
6037         my $volid = PVE::Storage::vdisk_alloc(
6038             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6039
6040         print STDERR "new volume ID is '$volid'\n";
6041         $d->{volid} = $volid;
6042
6043         PVE::Storage::activate_volumes($storecfg, [$volid]);
6044
6045         $map->{$virtdev} = $volid;
6046     }
6047
6048     return $map;
6049 };
6050
6051 sub restore_update_config_line {
6052     my ($cookie, $map, $line, $unique) = @_;
6053
6054     return '' if $line =~ m/^\#qmdump\#/;
6055     return '' if $line =~ m/^\#vzdump\#/;
6056     return '' if $line =~ m/^lock:/;
6057     return '' if $line =~ m/^unused\d+:/;
6058     return '' if $line =~ m/^parent:/;
6059
6060     my $res = '';
6061
6062     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6063     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6064         # try to convert old 1.X settings
6065         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6066         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6067             my ($model, $macaddr) = split(/\=/, $devconfig);
6068             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6069             my $net = {
6070                 model => $model,
6071                 bridge => "vmbr$ind",
6072                 macaddr => $macaddr,
6073             };
6074             my $netstr = print_net($net);
6075
6076             $res .= "net$cookie->{netcount}: $netstr\n";
6077             $cookie->{netcount}++;
6078         }
6079     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6080         my ($id, $netstr) = ($1, $2);
6081         my $net = parse_net($netstr);
6082         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6083         $netstr = print_net($net);
6084         $res .= "$id: $netstr\n";
6085     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
6086         my $virtdev = $1;
6087         my $value = $3;
6088         my $di = parse_drive($virtdev, $value);
6089         if (defined($di->{backup}) && !$di->{backup}) {
6090             $res .= "#$line";
6091         } elsif ($map->{$virtdev}) {
6092             delete $di->{format}; # format can change on restore
6093             $di->{file} = $map->{$virtdev};
6094             $value = print_drive($di);
6095             $res .= "$virtdev: $value\n";
6096         } else {
6097             $res .= $line;
6098         }
6099     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6100         my $vmgenid = $1;
6101         if ($vmgenid ne '0') {
6102             # always generate a new vmgenid if there was a valid one setup
6103             $vmgenid = generate_uuid();
6104         }
6105         $res .= "vmgenid: $vmgenid\n";
6106     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6107         my ($uuid, $uuid_str);
6108         UUID::generate($uuid);
6109         UUID::unparse($uuid, $uuid_str);
6110         my $smbios1 = parse_smbios1($2);
6111         $smbios1->{uuid} = $uuid_str;
6112         $res .= $1.print_smbios1($smbios1)."\n";
6113     } else {
6114         $res .= $line;
6115     }
6116
6117     return $res;
6118 }
6119
6120 my $restore_deactivate_volumes = sub {
6121     my ($storecfg, $devinfo) = @_;
6122
6123     my $vollist = [];
6124     foreach my $devname (keys %$devinfo) {
6125         my $volid = $devinfo->{$devname}->{volid};
6126         push @$vollist, $volid if $volid;
6127     }
6128
6129     PVE::Storage::deactivate_volumes($storecfg, $vollist);
6130 };
6131
6132 my $restore_destroy_volumes = sub {
6133     my ($storecfg, $devinfo) = @_;
6134
6135     foreach my $devname (keys %$devinfo) {
6136         my $volid = $devinfo->{$devname}->{volid};
6137         next if !$volid;
6138         eval {
6139             if ($volid =~ m|^/|) {
6140                 unlink $volid || die 'unlink failed\n';
6141             } else {
6142                 PVE::Storage::vdisk_free($storecfg, $volid);
6143             }
6144             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6145         };
6146         print STDERR "unable to cleanup '$volid' - $@" if $@;
6147     }
6148 };
6149
6150 sub scan_volids {
6151     my ($cfg, $vmid) = @_;
6152
6153     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6154
6155     my $volid_hash = {};
6156     foreach my $storeid (keys %$info) {
6157         foreach my $item (@{$info->{$storeid}}) {
6158             next if !($item->{volid} && $item->{size});
6159             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6160             $volid_hash->{$item->{volid}} = $item;
6161         }
6162     }
6163
6164     return $volid_hash;
6165 }
6166
6167 sub update_disk_config {
6168     my ($vmid, $conf, $volid_hash) = @_;
6169
6170     my $changes;
6171     my $prefix = "VM $vmid";
6172
6173     # used and unused disks
6174     my $referenced = {};
6175
6176     # Note: it is allowed to define multiple storages with same path (alias), so
6177     # we need to check both 'volid' and real 'path' (two different volid can point
6178     # to the same path).
6179
6180     my $referencedpath = {};
6181
6182     # update size info
6183     PVE::QemuConfig->foreach_volume($conf, sub {
6184         my ($opt, $drive) = @_;
6185
6186         my $volid = $drive->{file};
6187         return if !$volid;
6188         my $volume = $volid_hash->{$volid};
6189
6190         # mark volid as "in-use" for next step
6191         $referenced->{$volid} = 1;
6192         if ($volume && (my $path = $volume->{path})) {
6193             $referencedpath->{$path} = 1;
6194         }
6195
6196         return if drive_is_cdrom($drive);
6197         return if !$volume;
6198
6199         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6200         if (defined($updated)) {
6201             $changes = 1;
6202             $conf->{$opt} = print_drive($updated);
6203             print "$prefix ($opt): $msg\n";
6204         }
6205     });
6206
6207     # remove 'unusedX' entry if volume is used
6208     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6209         my ($opt, $drive) = @_;
6210
6211         my $volid = $drive->{file};
6212         return if !$volid;
6213
6214         my $path;
6215         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6216         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6217             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6218             $changes = 1;
6219             delete $conf->{$opt};
6220         }
6221
6222         $referenced->{$volid} = 1;
6223         $referencedpath->{$path} = 1 if $path;
6224     });
6225
6226     foreach my $volid (sort keys %$volid_hash) {
6227         next if $volid =~ m/vm-$vmid-state-/;
6228         next if $referenced->{$volid};
6229         my $path = $volid_hash->{$volid}->{path};
6230         next if !$path; # just to be sure
6231         next if $referencedpath->{$path};
6232         $changes = 1;
6233         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6234         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6235         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6236     }
6237
6238     return $changes;
6239 }
6240
6241 sub rescan {
6242     my ($vmid, $nolock, $dryrun) = @_;
6243
6244     my $cfg = PVE::Storage::config();
6245
6246     print "rescan volumes...\n";
6247     my $volid_hash = scan_volids($cfg, $vmid);
6248
6249     my $updatefn =  sub {
6250         my ($vmid) = @_;
6251
6252         my $conf = PVE::QemuConfig->load_config($vmid);
6253
6254         PVE::QemuConfig->check_lock($conf);
6255
6256         my $vm_volids = {};
6257         foreach my $volid (keys %$volid_hash) {
6258             my $info = $volid_hash->{$volid};
6259             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6260         }
6261
6262         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6263
6264         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6265     };
6266
6267     if (defined($vmid)) {
6268         if ($nolock) {
6269             &$updatefn($vmid);
6270         } else {
6271             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6272         }
6273     } else {
6274         my $vmlist = config_list();
6275         foreach my $vmid (keys %$vmlist) {
6276             if ($nolock) {
6277                 &$updatefn($vmid);
6278             } else {
6279                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6280             }
6281         }
6282     }
6283 }
6284
6285 sub restore_proxmox_backup_archive {
6286     my ($archive, $vmid, $user, $options) = @_;
6287
6288     my $storecfg = PVE::Storage::config();
6289
6290     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6291     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6292
6293     my $fingerprint = $scfg->{fingerprint};
6294     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6295
6296     my $repo = PVE::PBSClient::get_repository($scfg);
6297
6298     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6299     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6300     local $ENV{PBS_PASSWORD} = $password;
6301     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6302
6303     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6304         PVE::Storage::parse_volname($storecfg, $archive);
6305
6306     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6307
6308     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6309
6310     my $tmpdir = "/var/tmp/vzdumptmp$$";
6311     rmtree $tmpdir;
6312     mkpath $tmpdir;
6313
6314     my $conffile = PVE::QemuConfig->config_file($vmid);
6315      # disable interrupts (always do cleanups)
6316     local $SIG{INT} =
6317         local $SIG{TERM} =
6318         local $SIG{QUIT} =
6319         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6320
6321     # Note: $oldconf is undef if VM does not exists
6322     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6323     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6324     my $new_conf_raw = '';
6325
6326     my $rpcenv = PVE::RPCEnvironment::get();
6327     my $devinfo = {};
6328
6329     eval {
6330         # enable interrupts
6331         local $SIG{INT} =
6332             local $SIG{TERM} =
6333             local $SIG{QUIT} =
6334             local $SIG{HUP} =
6335             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6336
6337         my $cfgfn = "$tmpdir/qemu-server.conf";
6338         my $firewall_config_fn = "$tmpdir/fw.conf";
6339         my $index_fn = "$tmpdir/index.json";
6340
6341         my $cmd = "restore";
6342
6343         my $param = [$pbs_backup_name, "index.json", $index_fn];
6344         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6345         my $index = PVE::Tools::file_get_contents($index_fn);
6346         $index = decode_json($index);
6347
6348         # print Dumper($index);
6349         foreach my $info (@{$index->{files}}) {
6350             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6351                 my $devname = $1;
6352                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6353                     $devinfo->{$devname}->{size} = $1;
6354                 } else {
6355                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6356                 }
6357             }
6358         }
6359
6360         my $is_qemu_server_backup = scalar(
6361             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6362         );
6363         if (!$is_qemu_server_backup) {
6364             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6365         }
6366         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6367
6368         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6369         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6370
6371         if ($has_firewall_config) {
6372             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6373             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6374
6375             my $pve_firewall_dir = '/etc/pve/firewall';
6376             mkdir $pve_firewall_dir; # make sure the dir exists
6377             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6378         }
6379
6380         my $fh = IO::File->new($cfgfn, "r") ||
6381             die "unable to read qemu-server.conf - $!\n";
6382
6383         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6384
6385         # fixme: rate limit?
6386
6387         # create empty/temp config
6388         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6389
6390         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6391
6392         # allocate volumes
6393         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6394
6395         foreach my $virtdev (sort keys %$virtdev_hash) {
6396             my $d = $virtdev_hash->{$virtdev};
6397             next if $d->{is_cloudinit}; # no need to restore cloudinit
6398
6399             # for live-restore we only want to preload the efidisk
6400             next if $options->{live} && $virtdev ne 'efidisk0';
6401
6402             my $volid = $d->{volid};
6403
6404             my $path = PVE::Storage::path($storecfg, $volid);
6405
6406             my $pbs_restore_cmd = [
6407                 '/usr/bin/pbs-restore',
6408                 '--repository', $repo,
6409                 $pbs_backup_name,
6410                 "$d->{devname}.img.fidx",
6411                 $path,
6412                 '--verbose',
6413                 ];
6414
6415             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6416             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
6417
6418             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6419                 push @$pbs_restore_cmd, '--skip-zero';
6420             }
6421
6422             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6423             print "restore proxmox backup image: $dbg_cmdstring\n";
6424             run_command($pbs_restore_cmd);
6425         }
6426
6427         $fh->seek(0, 0) || die "seek failed - $!\n";
6428
6429         my $cookie = { netcount => 0 };
6430         while (defined(my $line = <$fh>)) {
6431             $new_conf_raw .= restore_update_config_line(
6432                 $cookie,
6433                 $map,
6434                 $line,
6435                 $options->{unique},
6436             );
6437         }
6438
6439         $fh->close();
6440     };
6441     my $err = $@;
6442
6443     if ($err || !$options->{live}) {
6444         $restore_deactivate_volumes->($storecfg, $devinfo);
6445     }
6446
6447     rmtree $tmpdir;
6448
6449     if ($err) {
6450         $restore_destroy_volumes->($storecfg, $devinfo);
6451         die $err;
6452     }
6453
6454     if ($options->{live}) {
6455         # keep lock during live-restore
6456         $new_conf_raw .= "\nlock: create";
6457     }
6458
6459     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6460
6461     PVE::Cluster::cfs_update(); # make sure we read new file
6462
6463     eval { rescan($vmid, 1); };
6464     warn $@ if $@;
6465
6466     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
6467
6468     if ($options->{live}) {
6469         # enable interrupts
6470         local $SIG{INT} =
6471             local $SIG{TERM} =
6472             local $SIG{QUIT} =
6473             local $SIG{HUP} =
6474             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
6475
6476         my $conf = PVE::QemuConfig->load_config($vmid);
6477         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
6478
6479         delete $devinfo->{'drive-efidisk0'}; # this special drive is already restored before start
6480         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $repo, $keyfile, $pbs_backup_name);
6481
6482         PVE::QemuConfig->remove_lock($vmid, "create");
6483     }
6484 }
6485
6486 sub pbs_live_restore {
6487     my ($vmid, $conf, $storecfg, $restored_disks, $repo, $keyfile, $snap) = @_;
6488
6489     print "starting VM for live-restore\n";
6490     print "repository: '$repo', snapshot: '$snap'\n";
6491
6492     my $pbs_backing = {};
6493     for my $ds (keys %$restored_disks) {
6494         $ds =~ m/^drive-(.*)$/;
6495         my $confname = $1;
6496         $pbs_backing->{$confname} = {
6497             repository => $repo,
6498             snapshot => $snap,
6499             archive => "$ds.img.fidx",
6500         };
6501         $pbs_backing->{$confname}->{keyfile} = $keyfile if -e $keyfile;
6502
6503         my $drive = parse_drive($confname, $conf->{$confname});
6504         print "restoring '$ds' to '$drive->{file}'\n";
6505     }
6506
6507     my $drives_streamed = 0;
6508     eval {
6509         # make sure HA doesn't interrupt our restore by stopping the VM
6510         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
6511             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
6512         }
6513
6514         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
6515         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
6516         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
6517
6518         my $qmeventd_fd = register_qmeventd_handle($vmid);
6519
6520         # begin streaming, i.e. data copy from PBS to target disk for every vol,
6521         # this will effectively collapse the backing image chain consisting of
6522         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
6523         # removes itself once all backing images vanish with 'auto-remove=on')
6524         my $jobs = {};
6525         for my $ds (sort keys %$restored_disks) {
6526             my $job_id = "restore-$ds";
6527             mon_cmd($vmid, 'block-stream',
6528                 'job-id' => $job_id,
6529                 device => "$ds",
6530             );
6531             $jobs->{$job_id} = {};
6532         }
6533
6534         mon_cmd($vmid, 'cont');
6535         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
6536
6537         print "restore-drive jobs finished successfully, removing all tracking block devices"
6538             ." to disconnect from Proxmox Backup Server\n";
6539
6540         for my $ds (sort keys %$restored_disks) {
6541             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
6542         }
6543
6544         close($qmeventd_fd);
6545     };
6546
6547     my $err = $@;
6548
6549     if ($err) {
6550         warn "An error occured during live-restore: $err\n";
6551         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
6552         die "live-restore failed\n";
6553     }
6554 }
6555
6556 sub restore_vma_archive {
6557     my ($archive, $vmid, $user, $opts, $comp) = @_;
6558
6559     my $readfrom = $archive;
6560
6561     my $cfg = PVE::Storage::config();
6562     my $commands = [];
6563     my $bwlimit = $opts->{bwlimit};
6564
6565     my $dbg_cmdstring = '';
6566     my $add_pipe = sub {
6567         my ($cmd) = @_;
6568         push @$commands, $cmd;
6569         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6570         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6571         $readfrom = '-';
6572     };
6573
6574     my $input = undef;
6575     if ($archive eq '-') {
6576         $input = '<&STDIN';
6577     } else {
6578         # If we use a backup from a PVE defined storage we also consider that
6579         # storage's rate limit:
6580         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6581         if (defined($volid)) {
6582             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6583             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6584             if ($readlimit) {
6585                 print STDERR "applying read rate limit: $readlimit\n";
6586                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6587                 $add_pipe->($cstream);
6588             }
6589         }
6590     }
6591
6592     if ($comp) {
6593         my $info = PVE::Storage::decompressor_info('vma', $comp);
6594         my $cmd = $info->{decompressor};
6595         push @$cmd, $readfrom;
6596         $add_pipe->($cmd);
6597     }
6598
6599     my $tmpdir = "/var/tmp/vzdumptmp$$";
6600     rmtree $tmpdir;
6601
6602     # disable interrupts (always do cleanups)
6603     local $SIG{INT} =
6604         local $SIG{TERM} =
6605         local $SIG{QUIT} =
6606         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6607
6608     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6609     POSIX::mkfifo($mapfifo, 0600);
6610     my $fifofh;
6611     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
6612
6613     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6614
6615     my $oldtimeout;
6616     my $timeout = 5;
6617
6618     my $devinfo = {};
6619
6620     my $rpcenv = PVE::RPCEnvironment::get();
6621
6622     my $conffile = PVE::QemuConfig->config_file($vmid);
6623
6624     # Note: $oldconf is undef if VM does not exist
6625     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6626     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6627     my $new_conf_raw = '';
6628
6629     my %storage_limits;
6630
6631     my $print_devmap = sub {
6632         my $cfgfn = "$tmpdir/qemu-server.conf";
6633
6634         # we can read the config - that is already extracted
6635         my $fh = IO::File->new($cfgfn, "r") ||
6636             die "unable to read qemu-server.conf - $!\n";
6637
6638         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6639         if (-f $fwcfgfn) {
6640             my $pve_firewall_dir = '/etc/pve/firewall';
6641             mkdir $pve_firewall_dir; # make sure the dir exists
6642             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6643         }
6644
6645         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6646
6647         foreach my $info (values %{$virtdev_hash}) {
6648             my $storeid = $info->{storeid};
6649             next if defined($storage_limits{$storeid});
6650
6651             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
6652             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
6653             $storage_limits{$storeid} = $limit * 1024;
6654         }
6655
6656         foreach my $devname (keys %$devinfo) {
6657             die "found no device mapping information for device '$devname'\n"
6658                 if !$devinfo->{$devname}->{virtdev};
6659         }
6660
6661         # create empty/temp config
6662         if ($oldconf) {
6663             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6664             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6665         }
6666
6667         # allocate volumes
6668         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6669
6670         # print restore information to $fifofh
6671         foreach my $virtdev (sort keys %$virtdev_hash) {
6672             my $d = $virtdev_hash->{$virtdev};
6673             next if $d->{is_cloudinit}; # no need to restore cloudinit
6674
6675             my $storeid = $d->{storeid};
6676             my $volid = $d->{volid};
6677
6678             my $map_opts = '';
6679             if (my $limit = $storage_limits{$storeid}) {
6680                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6681             }
6682
6683             my $write_zeros = 1;
6684             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6685                 $write_zeros = 0;
6686             }
6687
6688             my $path = PVE::Storage::path($cfg, $volid);
6689
6690             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6691
6692             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6693         }
6694
6695         $fh->seek(0, 0) || die "seek failed - $!\n";
6696
6697         my $cookie = { netcount => 0 };
6698         while (defined(my $line = <$fh>)) {
6699             $new_conf_raw .= restore_update_config_line(
6700                 $cookie,
6701                 $map,
6702                 $line,
6703                 $opts->{unique},
6704             );
6705         }
6706
6707         $fh->close();
6708     };
6709
6710     eval {
6711         # enable interrupts
6712         local $SIG{INT} =
6713             local $SIG{TERM} =
6714             local $SIG{QUIT} =
6715             local $SIG{HUP} =
6716             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6717         local $SIG{ALRM} = sub { die "got timeout\n"; };
6718
6719         $oldtimeout = alarm($timeout);
6720
6721         my $parser = sub {
6722             my $line = shift;
6723
6724             print "$line\n";
6725
6726             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6727                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6728                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6729             } elsif ($line =~ m/^CTIME: /) {
6730                 # we correctly received the vma config, so we can disable
6731                 # the timeout now for disk allocation (set to 10 minutes, so
6732                 # that we always timeout if something goes wrong)
6733                 alarm(600);
6734                 &$print_devmap();
6735                 print $fifofh "done\n";
6736                 my $tmp = $oldtimeout || 0;
6737                 $oldtimeout = undef;
6738                 alarm($tmp);
6739                 close($fifofh);
6740                 $fifofh = undef;
6741             }
6742         };
6743
6744         print "restore vma archive: $dbg_cmdstring\n";
6745         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6746     };
6747     my $err = $@;
6748
6749     alarm($oldtimeout) if $oldtimeout;
6750
6751     $restore_deactivate_volumes->($cfg, $devinfo);
6752
6753     close($fifofh) if $fifofh;
6754     unlink $mapfifo;
6755     rmtree $tmpdir;
6756
6757     if ($err) {
6758         $restore_destroy_volumes->($cfg, $devinfo);
6759         die $err;
6760     }
6761
6762     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6763
6764     PVE::Cluster::cfs_update(); # make sure we read new file
6765
6766     eval { rescan($vmid, 1); };
6767     warn $@ if $@;
6768
6769     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
6770 }
6771
6772 sub restore_tar_archive {
6773     my ($archive, $vmid, $user, $opts) = @_;
6774
6775     if ($archive ne '-') {
6776         my $firstfile = tar_archive_read_firstfile($archive);
6777         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
6778             if $firstfile ne 'qemu-server.conf';
6779     }
6780
6781     my $storecfg = PVE::Storage::config();
6782
6783     # avoid zombie disks when restoring over an existing VM -> cleanup first
6784     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6785     # skiplock=1 because qmrestore has set the 'create' lock itself already
6786     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6787     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6788
6789     my $tocmd = "/usr/lib/qemu-server/qmextract";
6790
6791     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6792     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6793     $tocmd .= ' --prealloc' if $opts->{prealloc};
6794     $tocmd .= ' --info' if $opts->{info};
6795
6796     # tar option "xf" does not autodetect compression when read from STDIN,
6797     # so we pipe to zcat
6798     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6799         PVE::Tools::shellquote("--to-command=$tocmd");
6800
6801     my $tmpdir = "/var/tmp/vzdumptmp$$";
6802     mkpath $tmpdir;
6803
6804     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6805     local $ENV{VZDUMP_VMID} = $vmid;
6806     local $ENV{VZDUMP_USER} = $user;
6807
6808     my $conffile = PVE::QemuConfig->config_file($vmid);
6809     my $new_conf_raw = '';
6810
6811     # disable interrupts (always do cleanups)
6812     local $SIG{INT} =
6813         local $SIG{TERM} =
6814         local $SIG{QUIT} =
6815         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6816
6817     eval {
6818         # enable interrupts
6819         local $SIG{INT} =
6820             local $SIG{TERM} =
6821             local $SIG{QUIT} =
6822             local $SIG{HUP} =
6823             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6824
6825         if ($archive eq '-') {
6826             print "extracting archive from STDIN\n";
6827             run_command($cmd, input => "<&STDIN");
6828         } else {
6829             print "extracting archive '$archive'\n";
6830             run_command($cmd);
6831         }
6832
6833         return if $opts->{info};
6834
6835         # read new mapping
6836         my $map = {};
6837         my $statfile = "$tmpdir/qmrestore.stat";
6838         if (my $fd = IO::File->new($statfile, "r")) {
6839             while (defined (my $line = <$fd>)) {
6840                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6841                     $map->{$1} = $2 if $1;
6842                 } else {
6843                     print STDERR "unable to parse line in statfile - $line\n";
6844                 }
6845             }
6846             $fd->close();
6847         }
6848
6849         my $confsrc = "$tmpdir/qemu-server.conf";
6850
6851         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
6852
6853         my $cookie = { netcount => 0 };
6854         while (defined (my $line = <$srcfd>)) {
6855             $new_conf_raw .= restore_update_config_line(
6856                 $cookie,
6857                 $map,
6858                 $line,
6859                 $opts->{unique},
6860             );
6861         }
6862
6863         $srcfd->close();
6864     };
6865     if (my $err = $@) {
6866         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6867         die $err;
6868     }
6869
6870     rmtree $tmpdir;
6871
6872     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6873
6874     PVE::Cluster::cfs_update(); # make sure we read new file
6875
6876     eval { rescan($vmid, 1); };
6877     warn $@ if $@;
6878 };
6879
6880 sub foreach_storage_used_by_vm {
6881     my ($conf, $func) = @_;
6882
6883     my $sidhash = {};
6884
6885     PVE::QemuConfig->foreach_volume($conf, sub {
6886         my ($ds, $drive) = @_;
6887         return if drive_is_cdrom($drive);
6888
6889         my $volid = $drive->{file};
6890
6891         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6892         $sidhash->{$sid} = $sid if $sid;
6893     });
6894
6895     foreach my $sid (sort keys %$sidhash) {
6896         &$func($sid);
6897     }
6898 }
6899
6900 my $qemu_snap_storage = {
6901     rbd => 1,
6902 };
6903 sub do_snapshots_with_qemu {
6904     my ($storecfg, $volid) = @_;
6905
6906     my $storage_name = PVE::Storage::parse_volume_id($volid);
6907     my $scfg = $storecfg->{ids}->{$storage_name};
6908     die "could not find storage '$storage_name'\n" if !defined($scfg);
6909
6910     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6911         return 1;
6912     }
6913
6914     if ($volid =~ m/\.(qcow2|qed)$/){
6915         return 1;
6916     }
6917
6918     return;
6919 }
6920
6921 sub qga_check_running {
6922     my ($vmid, $nowarn) = @_;
6923
6924     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6925     if ($@) {
6926         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6927         return 0;
6928     }
6929     return 1;
6930 }
6931
6932 sub template_create {
6933     my ($vmid, $conf, $disk) = @_;
6934
6935     my $storecfg = PVE::Storage::config();
6936
6937     PVE::QemuConfig->foreach_volume($conf, sub {
6938         my ($ds, $drive) = @_;
6939
6940         return if drive_is_cdrom($drive);
6941         return if $disk && $ds ne $disk;
6942
6943         my $volid = $drive->{file};
6944         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6945
6946         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6947         $drive->{file} = $voliddst;
6948         $conf->{$ds} = print_drive($drive);
6949         PVE::QemuConfig->write_config($vmid, $conf);
6950     });
6951 }
6952
6953 sub convert_iscsi_path {
6954     my ($path) = @_;
6955
6956     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6957         my $portal = $1;
6958         my $target = $2;
6959         my $lun = $3;
6960
6961         my $initiator_name = get_initiator_name();
6962
6963         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6964                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6965     }
6966
6967     die "cannot convert iscsi path '$path', unkown format\n";
6968 }
6969
6970 sub qemu_img_convert {
6971     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6972
6973     my $storecfg = PVE::Storage::config();
6974     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6975     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6976
6977     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6978
6979     my $cachemode;
6980     my $src_path;
6981     my $src_is_iscsi = 0;
6982     my $src_format;
6983
6984     if ($src_storeid) {
6985         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6986         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6987         $src_format = qemu_img_format($src_scfg, $src_volname);
6988         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6989         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6990         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6991     } elsif (-f $src_volid) {
6992         $src_path = $src_volid;
6993         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6994             $src_format = $1;
6995         }
6996     }
6997
6998     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6999
7000     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7001     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
7002     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7003     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
7004
7005     my $cmd = [];
7006     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7007     push @$cmd, '-l', "snapshot.name=$snapname"
7008         if $snapname && $src_format && $src_format eq "qcow2";
7009     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
7010     push @$cmd, '-T', $cachemode if defined($cachemode);
7011
7012     if ($src_is_iscsi) {
7013         push @$cmd, '--image-opts';
7014         $src_path = convert_iscsi_path($src_path);
7015     } elsif ($src_format) {
7016         push @$cmd, '-f', $src_format;
7017     }
7018
7019     if ($dst_is_iscsi) {
7020         push @$cmd, '--target-image-opts';
7021         $dst_path = convert_iscsi_path($dst_path);
7022     } else {
7023         push @$cmd, '-O', $dst_format;
7024     }
7025
7026     push @$cmd, $src_path;
7027
7028     if (!$dst_is_iscsi && $is_zero_initialized) {
7029         push @$cmd, "zeroinit:$dst_path";
7030     } else {
7031         push @$cmd, $dst_path;
7032     }
7033
7034     my $parser = sub {
7035         my $line = shift;
7036         if($line =~ m/\((\S+)\/100\%\)/){
7037             my $percent = $1;
7038             my $transferred = int($size * $percent / 100);
7039             my $total_h = render_bytes($size, 1);
7040             my $transferred_h = render_bytes($transferred, 1);
7041
7042             print "transferred $transferred_h of $total_h ($percent%)\n";
7043         }
7044
7045     };
7046
7047     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7048     my $err = $@;
7049     die "copy failed: $err" if $err;
7050 }
7051
7052 sub qemu_img_format {
7053     my ($scfg, $volname) = @_;
7054
7055     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7056         return $1;
7057     } else {
7058         return "raw";
7059     }
7060 }
7061
7062 sub qemu_drive_mirror {
7063     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7064
7065     $jobs = {} if !$jobs;
7066
7067     my $qemu_target;
7068     my $format;
7069     $jobs->{"drive-$drive"} = {};
7070
7071     if ($dst_volid =~ /^nbd:/) {
7072         $qemu_target = $dst_volid;
7073         $format = "nbd";
7074     } else {
7075         my $storecfg = PVE::Storage::config();
7076         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7077
7078         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7079
7080         $format = qemu_img_format($dst_scfg, $dst_volname);
7081
7082         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7083
7084         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7085     }
7086
7087     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7088     $opts->{format} = $format if $format;
7089
7090     if (defined($src_bitmap)) {
7091         $opts->{sync} = 'incremental';
7092         $opts->{bitmap} = $src_bitmap;
7093         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7094     }
7095
7096     if (defined($bwlimit)) {
7097         $opts->{speed} = $bwlimit * 1024;
7098         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7099     } else {
7100         print "drive mirror is starting for drive-$drive\n";
7101     }
7102
7103     # if a job already runs for this device we get an error, catch it for cleanup
7104     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7105     if (my $err = $@) {
7106         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7107         warn "$@\n" if $@;
7108         die "mirroring error: $err\n";
7109     }
7110
7111     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7112 }
7113
7114 # $completion can be either
7115 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7116 # 'cancel': wait until all jobs are ready, block-job-cancel them
7117 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7118 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7119 sub qemu_drive_mirror_monitor {
7120     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7121
7122     $completion //= 'complete';
7123     $op //= "mirror";
7124
7125     eval {
7126         my $err_complete = 0;
7127
7128         my $starttime = time ();
7129         while (1) {
7130             die "block job ('$op') timed out\n" if $err_complete > 300;
7131
7132             my $stats = mon_cmd($vmid, "query-block-jobs");
7133             my $ctime = time();
7134
7135             my $running_jobs = {};
7136             for my $stat (@$stats) {
7137                 next if $stat->{type} ne $op;
7138                 $running_jobs->{$stat->{device}} = $stat;
7139             }
7140
7141             my $readycounter = 0;
7142
7143             for my $job_id (sort keys %$jobs) {
7144                 my $job = $running_jobs->{$job_id};
7145
7146                 my $vanished = !defined($job);
7147                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7148                 if($complete || ($vanished && $completion eq 'auto')) {
7149                     print "$job_id: $op-job finished\n";
7150                     delete $jobs->{$job_id};
7151                     next;
7152                 }
7153
7154                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7155
7156                 my $busy = $job->{busy};
7157                 my $ready = $job->{ready};
7158                 if (my $total = $job->{len}) {
7159                     my $transferred = $job->{offset} || 0;
7160                     my $remaining = $total - $transferred;
7161                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7162
7163                     my $duration = $ctime - $starttime;
7164                     my $total_h = render_bytes($total, 1);
7165                     my $transferred_h = render_bytes($transferred, 1);
7166
7167                     my $status = sprintf(
7168                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7169                         render_duration($duration),
7170                     );
7171
7172                     if ($ready) {
7173                         if ($busy) {
7174                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7175                         } else {
7176                             $status .= ", ready";
7177                         }
7178                     }
7179                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7180                     $jobs->{$job_id}->{ready} = $ready;
7181                 }
7182
7183                 $readycounter++ if $job->{ready};
7184             }
7185
7186             last if scalar(keys %$jobs) == 0;
7187
7188             if ($readycounter == scalar(keys %$jobs)) {
7189                 print "all '$op' jobs are ready\n";
7190
7191                 # do the complete later (or has already been done)
7192                 last if $completion eq 'skip' || $completion eq 'auto';
7193
7194                 if ($vmiddst && $vmiddst != $vmid) {
7195                     my $agent_running = $qga && qga_check_running($vmid);
7196                     if ($agent_running) {
7197                         print "freeze filesystem\n";
7198                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7199                     } else {
7200                         print "suspend vm\n";
7201                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7202                     }
7203
7204                     # if we clone a disk for a new target vm, we don't switch the disk
7205                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7206
7207                     if ($agent_running) {
7208                         print "unfreeze filesystem\n";
7209                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7210                     } else {
7211                         print "resume vm\n";
7212                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
7213                     }
7214
7215                     last;
7216                 } else {
7217
7218                     for my $job_id (sort keys %$jobs) {
7219                         # try to switch the disk if source and destination are on the same guest
7220                         print "$job_id: Completing block job_id...\n";
7221
7222                         my $op;
7223                         if ($completion eq 'complete') {
7224                             $op = 'block-job-complete';
7225                         } elsif ($completion eq 'cancel') {
7226                             $op = 'block-job-cancel';
7227                         } else {
7228                             die "invalid completion value: $completion\n";
7229                         }
7230                         eval { mon_cmd($vmid, $op, device => $job_id) };
7231                         if ($@ =~ m/cannot be completed/) {
7232                             print "$job_id: block job cannot be completed, trying again.\n";
7233                             $err_complete++;
7234                         }else {
7235                             print "$job_id: Completed successfully.\n";
7236                             $jobs->{$job_id}->{complete} = 1;
7237                         }
7238                     }
7239                 }
7240             }
7241             sleep 1;
7242         }
7243     };
7244     my $err = $@;
7245
7246     if ($err) {
7247         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7248         die "block job ($op) error: $err";
7249     }
7250 }
7251
7252 sub qemu_blockjobs_cancel {
7253     my ($vmid, $jobs) = @_;
7254
7255     foreach my $job (keys %$jobs) {
7256         print "$job: Cancelling block job\n";
7257         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7258         $jobs->{$job}->{cancel} = 1;
7259     }
7260
7261     while (1) {
7262         my $stats = mon_cmd($vmid, "query-block-jobs");
7263
7264         my $running_jobs = {};
7265         foreach my $stat (@$stats) {
7266             $running_jobs->{$stat->{device}} = $stat;
7267         }
7268
7269         foreach my $job (keys %$jobs) {
7270
7271             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7272                 print "$job: Done.\n";
7273                 delete $jobs->{$job};
7274             }
7275         }
7276
7277         last if scalar(keys %$jobs) == 0;
7278
7279         sleep 1;
7280     }
7281 }
7282
7283 sub clone_disk {
7284     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7285         $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
7286
7287     my $newvolid;
7288
7289     if (!$full) {
7290         print "create linked clone of drive $drivename ($drive->{file})\n";
7291         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7292         push @$newvollist, $newvolid;
7293     } else {
7294
7295         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7296         $storeid = $storage if $storage;
7297
7298         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7299
7300         print "create full clone of drive $drivename ($drive->{file})\n";
7301         my $name = undef;
7302         my $size = undef;
7303         if (drive_is_cloudinit($drive)) {
7304             $name = "vm-$newvmid-cloudinit";
7305             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7306             if ($scfg->{path}) {
7307                 $name .= ".$dst_format";
7308             }
7309             $snapname = undef;
7310             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7311         } elsif ($drivename eq 'efidisk0') {
7312             $size = get_efivars_size($conf);
7313         } else {
7314             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
7315         }
7316         $newvolid = PVE::Storage::vdisk_alloc(
7317             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
7318         );
7319         push @$newvollist, $newvolid;
7320
7321         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7322
7323         if (drive_is_cloudinit($drive)) {
7324             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
7325             # if this is the case, we have to complete any block-jobs still there from
7326             # previous drive-mirrors
7327             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
7328                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
7329             }
7330             goto no_data_clone;
7331         }
7332
7333         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7334         if (!$running || $snapname) {
7335             # TODO: handle bwlimits
7336             if ($drivename eq 'efidisk0') {
7337                 # the relevant data on the efidisk may be smaller than the source
7338                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
7339                 # that is given by the OVMF_VARS.fd
7340                 my $src_path = PVE::Storage::path($storecfg, $drive->{file});
7341                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
7342
7343                 # better for Ceph if block size is not too small, see bug #3324
7344                 my $bs = 1024*1024;
7345
7346                 run_command(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=$bs", "osize=$size",
7347                     "if=$src_path", "of=$dst_path"]);
7348             } else {
7349                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7350             }
7351         } else {
7352
7353             my $kvmver = get_running_qemu_version ($vmid);
7354             if (!min_version($kvmver, 2, 7)) {
7355                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7356                     if $drive->{iothread};
7357             }
7358
7359             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7360                 $completion, $qga, $bwlimit);
7361         }
7362     }
7363
7364 no_data_clone:
7365     my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
7366
7367     my $disk = $drive;
7368     $disk->{format} = undef;
7369     $disk->{file} = $newvolid;
7370     $disk->{size} = $size if defined($size);
7371
7372     return $disk;
7373 }
7374
7375 sub get_running_qemu_version {
7376     my ($vmid) = @_;
7377     my $res = mon_cmd($vmid, "query-version");
7378     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7379 }
7380
7381 sub qemu_use_old_bios_files {
7382     my ($machine_type) = @_;
7383
7384     return if !$machine_type;
7385
7386     my $use_old_bios_files = undef;
7387
7388     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7389         $machine_type = $1;
7390         $use_old_bios_files = 1;
7391     } else {
7392         my $version = extract_version($machine_type, kvm_user_version());
7393         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7394         # load new efi bios files on migration. So this hack is required to allow
7395         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7396         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7397         $use_old_bios_files = !min_version($version, 2, 4);
7398     }
7399
7400     return ($use_old_bios_files, $machine_type);
7401 }
7402
7403 sub get_efivars_size {
7404     my ($conf) = @_;
7405     my $arch = get_vm_arch($conf);
7406     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7407     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
7408     return -s $ovmf_vars;
7409 }
7410
7411 sub update_efidisk_size {
7412     my ($conf) = @_;
7413
7414     return if !defined($conf->{efidisk0});
7415
7416     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
7417     $disk->{size} = get_efivars_size($conf);
7418     $conf->{efidisk0} = print_drive($disk);
7419
7420     return;
7421 }
7422
7423 sub create_efidisk($$$$$) {
7424     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7425
7426     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7427     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7428
7429     my $vars_size_b = -s $ovmf_vars;
7430     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7431     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7432     PVE::Storage::activate_volumes($storecfg, [$volid]);
7433
7434     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7435     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7436
7437     return ($volid, $size/1024);
7438 }
7439
7440 sub vm_iothreads_list {
7441     my ($vmid) = @_;
7442
7443     my $res = mon_cmd($vmid, 'query-iothreads');
7444
7445     my $iothreads = {};
7446     foreach my $iothread (@$res) {
7447         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7448     }
7449
7450     return $iothreads;
7451 }
7452
7453 sub scsihw_infos {
7454     my ($conf, $drive) = @_;
7455
7456     my $maxdev = 0;
7457
7458     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7459         $maxdev = 7;
7460     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7461         $maxdev = 1;
7462     } else {
7463         $maxdev = 256;
7464     }
7465
7466     my $controller = int($drive->{index} / $maxdev);
7467     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
7468         ? "virtioscsi"
7469         : "scsihw";
7470
7471     return ($maxdev, $controller, $controller_prefix);
7472 }
7473
7474 sub windows_version {
7475     my ($ostype) = @_;
7476
7477     return 0 if !$ostype;
7478
7479     my $winversion = 0;
7480
7481     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7482         $winversion = 5;
7483     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7484         $winversion = 6;
7485     } elsif ($ostype =~ m/^win(\d+)$/) {
7486         $winversion = $1;
7487     }
7488
7489     return $winversion;
7490 }
7491
7492 sub resolve_dst_disk_format {
7493         my ($storecfg, $storeid, $src_volname, $format) = @_;
7494         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7495
7496         if (!$format) {
7497             # if no target format is specified, use the source disk format as hint
7498             if ($src_volname) {
7499                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7500                 $format = qemu_img_format($scfg, $src_volname);
7501             } else {
7502                 return $defFormat;
7503             }
7504         }
7505
7506         # test if requested format is supported - else use default
7507         my $supported = grep { $_ eq $format } @$validFormats;
7508         $format = $defFormat if !$supported;
7509         return $format;
7510 }
7511
7512 # NOTE: if this logic changes, please update docs & possibly gui logic
7513 sub find_vmstate_storage {
7514     my ($conf, $storecfg) = @_;
7515
7516     # first, return storage from conf if set
7517     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7518
7519     my ($target, $shared, $local);
7520
7521     foreach_storage_used_by_vm($conf, sub {
7522         my ($sid) = @_;
7523         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7524         my $dst = $scfg->{shared} ? \$shared : \$local;
7525         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7526     });
7527
7528     # second, use shared storage where VM has at least one disk
7529     # third, use local storage where VM has at least one disk
7530     # fall back to local storage
7531     $target = $shared // $local // 'local';
7532
7533     return $target;
7534 }
7535
7536 sub generate_uuid {
7537     my ($uuid, $uuid_str);
7538     UUID::generate($uuid);
7539     UUID::unparse($uuid, $uuid_str);
7540     return $uuid_str;
7541 }
7542
7543 sub generate_smbios1_uuid {
7544     return "uuid=".generate_uuid();
7545 }
7546
7547 sub nbd_stop {
7548     my ($vmid) = @_;
7549
7550     mon_cmd($vmid, 'nbd-server-stop');
7551 }
7552
7553 sub create_reboot_request {
7554     my ($vmid) = @_;
7555     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7556         or die "failed to create reboot trigger file: $!\n";
7557     close($fh);
7558 }
7559
7560 sub clear_reboot_request {
7561     my ($vmid) = @_;
7562     my $path = "/run/qemu-server/$vmid.reboot";
7563     my $res = 0;
7564
7565     $res = unlink($path);
7566     die "could not remove reboot request for $vmid: $!"
7567         if !$res && $! != POSIX::ENOENT;
7568
7569     return $res;
7570 }
7571
7572 sub bootorder_from_legacy {
7573     my ($conf, $bootcfg) = @_;
7574
7575     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
7576     my $bootindex_hash = {};
7577     my $i = 1;
7578     foreach my $o (split(//, $boot)) {
7579         $bootindex_hash->{$o} = $i*100;
7580         $i++;
7581     }
7582
7583     my $bootorder = {};
7584
7585     PVE::QemuConfig->foreach_volume($conf, sub {
7586         my ($ds, $drive) = @_;
7587
7588         if (drive_is_cdrom ($drive, 1)) {
7589             if ($bootindex_hash->{d}) {
7590                 $bootorder->{$ds} = $bootindex_hash->{d};
7591                 $bootindex_hash->{d} += 1;
7592             }
7593         } elsif ($bootindex_hash->{c}) {
7594             $bootorder->{$ds} = $bootindex_hash->{c}
7595                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
7596             $bootindex_hash->{c} += 1;
7597         }
7598     });
7599
7600     if ($bootindex_hash->{n}) {
7601         for (my $i = 0; $i < $MAX_NETS; $i++) {
7602             my $netname = "net$i";
7603             next if !$conf->{$netname};
7604             $bootorder->{$netname} = $bootindex_hash->{n};
7605             $bootindex_hash->{n} += 1;
7606         }
7607     }
7608
7609     return $bootorder;
7610 }
7611
7612 # Generate default device list for 'boot: order=' property. Matches legacy
7613 # default boot order, but with explicit device names. This is important, since
7614 # the fallback for when neither 'order' nor the old format is specified relies
7615 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
7616 sub get_default_bootdevices {
7617     my ($conf) = @_;
7618
7619     my @ret = ();
7620
7621     # harddisk
7622     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
7623     push @ret, $first if $first;
7624
7625     # cdrom
7626     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
7627     push @ret, $first if $first;
7628
7629     # network
7630     for (my $i = 0; $i < $MAX_NETS; $i++) {
7631         my $netname = "net$i";
7632         next if !$conf->{$netname};
7633         push @ret, $netname;
7634         last;
7635     }
7636
7637     return \@ret;
7638 }
7639
7640 sub device_bootorder {
7641     my ($conf) = @_;
7642
7643     return bootorder_from_legacy($conf) if !defined($conf->{boot});
7644
7645     my $boot = parse_property_string($boot_fmt, $conf->{boot});
7646
7647     my $bootorder = {};
7648     if (!defined($boot) || $boot->{legacy}) {
7649         $bootorder = bootorder_from_legacy($conf, $boot);
7650     } elsif ($boot->{order}) {
7651         my $i = 100; # start at 100 to allow user to insert devices before us with -args
7652         for my $dev (PVE::Tools::split_list($boot->{order})) {
7653             $bootorder->{$dev} = $i++;
7654         }
7655     }
7656
7657     return $bootorder;
7658 }
7659
7660 sub register_qmeventd_handle {
7661     my ($vmid) = @_;
7662
7663     my $fh;
7664     my $peer = "/var/run/qmeventd.sock";
7665     my $count = 0;
7666
7667     for (;;) {
7668         $count++;
7669         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
7670         last if $fh;
7671         if ($! != EINTR && $! != EAGAIN) {
7672             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
7673         }
7674         if ($count > 4) {
7675             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
7676               . "after $count retries\n";
7677         }
7678         usleep(25000);
7679     }
7680
7681     # send handshake to mark VM as backing up
7682     print $fh to_json({vzdump => {vmid => "$vmid"}});
7683
7684     # return handle to be closed later when inhibit is no longer required
7685     return $fh;
7686 }
7687
7688 # bash completion helper
7689
7690 sub complete_backup_archives {
7691     my ($cmdname, $pname, $cvalue) = @_;
7692
7693     my $cfg = PVE::Storage::config();
7694
7695     my $storeid;
7696
7697     if ($cvalue =~ m/^([^:]+):/) {
7698         $storeid = $1;
7699     }
7700
7701     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7702
7703     my $res = [];
7704     foreach my $id (keys %$data) {
7705         foreach my $item (@{$data->{$id}}) {
7706             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
7707             push @$res, $item->{volid} if defined($item->{volid});
7708         }
7709     }
7710
7711     return $res;
7712 }
7713
7714 my $complete_vmid_full = sub {
7715     my ($running) = @_;
7716
7717     my $idlist = vmstatus();
7718
7719     my $res = [];
7720
7721     foreach my $id (keys %$idlist) {
7722         my $d = $idlist->{$id};
7723         if (defined($running)) {
7724             next if $d->{template};
7725             next if $running && $d->{status} ne 'running';
7726             next if !$running && $d->{status} eq 'running';
7727         }
7728         push @$res, $id;
7729
7730     }
7731     return $res;
7732 };
7733
7734 sub complete_vmid {
7735     return &$complete_vmid_full();
7736 }
7737
7738 sub complete_vmid_stopped {
7739     return &$complete_vmid_full(0);
7740 }
7741
7742 sub complete_vmid_running {
7743     return &$complete_vmid_full(1);
7744 }
7745
7746 sub complete_storage {
7747
7748     my $cfg = PVE::Storage::config();
7749     my $ids = $cfg->{ids};
7750
7751     my $res = [];
7752     foreach my $sid (keys %$ids) {
7753         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7754         next if !$ids->{$sid}->{content}->{images};
7755         push @$res, $sid;
7756     }
7757
7758     return $res;
7759 }
7760
7761 sub complete_migration_storage {
7762     my ($cmd, $param, $current_value, $all_args) = @_;
7763
7764     my $targetnode = @$all_args[1];
7765
7766     my $cfg = PVE::Storage::config();
7767     my $ids = $cfg->{ids};
7768
7769     my $res = [];
7770     foreach my $sid (keys %$ids) {
7771         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7772         next if !$ids->{$sid}->{content}->{images};
7773         push @$res, $sid;
7774     }
7775
7776     return $res;
7777 }
7778
7779 sub vm_is_paused {
7780     my ($vmid) = @_;
7781     my $qmpstatus = eval {
7782         PVE::QemuConfig::assert_config_exists_on_node($vmid);
7783         mon_cmd($vmid, "query-status");
7784     };
7785     warn "$@\n" if $@;
7786     return $qmpstatus && $qmpstatus->{status} eq "paused";
7787 }
7788
7789 sub check_volume_storage_type {
7790     my ($storecfg, $vol) = @_;
7791
7792     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
7793     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7794     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
7795
7796     die "storage '$storeid' does not support content-type '$vtype'\n"
7797         if !$scfg->{content}->{$vtype};
7798
7799     return 1;
7800 }
7801
7802 1;