PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::CGroup;
  31 use PVE::DataCenterConfig;
  32 use PVE::Exception qw(raise raise_param_exc);
  33 use PVE::Format qw(render_duration render_bytes);
  34 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  35 use PVE::INotify;
  36 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  37 use PVE::ProcFSTools;
  38 use PVE::PBSClient;
  39 use PVE::RPCEnvironment;
  40 use PVE::Storage;
  41 use PVE::SysFSTools;
  42 use PVE::Systemd;
  43 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  44
  45 use PVE::QMPClient;
  46 use PVE::QemuConfig;
  47 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  48 use PVE::QemuServer::Cloudinit;
  49 use PVE::QemuServer::CGroup;
  50 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  51 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  52 use PVE::QemuServer::Machine;
  53 use PVE::QemuServer::Memory;
  54 use PVE::QemuServer::Monitor qw(mon_cmd);
  55 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  56 use PVE::QemuServer::USB qw(parse_usb_device);
  57
  58 my $have_sdn;
  59 eval {
  60     require PVE::Network::SDN::Zones;
  61     $have_sdn = 1;
  62 };
  63
  64 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  65 my $OVMF = {
  66     x86_64 => [
  67         "$EDK2_FW_BASE/OVMF_CODE.fd",
  68         "$EDK2_FW_BASE/OVMF_VARS.fd"
  69     ],
  70     aarch64 => [
  71         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  72         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  73     ],
  74 };
  75
  76 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  77
  78 # Note about locking: we use flock on the config file protect
  79 # against concurent actions.
  80 # Aditionaly, we have a 'lock' setting in the config file. This
  81 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  82 # allowed when such lock is set. But you can ignore this kind of
  83 # lock with the --skiplock flag.
  84
  85 cfs_register_file('/qemu-server/',
  86                   \&parse_vm_config,
  87                   \&write_vm_config);
  88
  89 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  90     description => "Some command save/restore state from this location.",
  91     type => 'string',
  92     maxLength => 128,
  93     optional => 1,
  94 });
  95
  96 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  97         description => "Specifies the Qemu machine type.",
  98         type => 'string',
  99         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 100         maxLength => 40,
 101         optional => 1,
 102 });
 103
 104
 105 sub map_storage {
 106     my ($map, $source) = @_;
 107
 108     return $source if !defined($map);
 109
 110     return $map->{entries}->{$source}
 111         if $map->{entries} && defined($map->{entries}->{$source});
 112
 113     return $map->{default} if $map->{default};
 114
 115     # identity (fallback)
 116     return $source;
 117 }
 118
 119 PVE::JSONSchema::register_standard_option('pve-targetstorage', {
 120     description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
 121     type => 'string',
 122     format => 'storagepair-list',
 123     optional => 1,
 124 });
 125
 126 #no warnings 'redefine';
 127
 128 my $nodename_cache;
 129 sub nodename {
 130     $nodename_cache //= PVE::INotify::nodename();
 131     return $nodename_cache;
 132 }
 133
 134 my $watchdog_fmt = {
 135     model => {
 136         default_key => 1,
 137         type => 'string',
 138         enum => [qw(i6300esb ib700)],
 139         description => "Watchdog type to emulate.",
 140         default => 'i6300esb',
 141         optional => 1,
 142     },
 143     action => {
 144         type => 'string',
 145         enum => [qw(reset shutdown poweroff pause debug none)],
 146         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 147         optional => 1,
 148     },
 149 };
 150 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 151
 152 my $agent_fmt = {
 153     enabled => {
 154         description => "Enable/disable Qemu GuestAgent.",
 155         type => 'boolean',
 156         default => 0,
 157         default_key => 1,
 158     },
 159     fstrim_cloned_disks => {
 160         description => "Run fstrim after moving a disk or migrating the VM.",
 161         type => 'boolean',
 162         optional => 1,
 163         default => 0
 164     },
 165     type => {
 166         description => "Select the agent type",
 167         type => 'string',
 168         default => 'virtio',
 169         optional => 1,
 170         enum => [qw(virtio isa)],
 171     },
 172 };
 173
 174 my $vga_fmt = {
 175     type => {
 176         description => "Select the VGA type.",
 177         type => 'string',
 178         default => 'std',
 179         optional => 1,
 180         default_key => 1,
 181         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 182     },
 183     memory => {
 184         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 185         type => 'integer',
 186         optional => 1,
 187         minimum => 4,
 188         maximum => 512,
 189     },
 190 };
 191
 192 my $ivshmem_fmt = {
 193     size => {
 194         type => 'integer',
 195         minimum => 1,
 196         description => "The size of the file in MB.",
 197     },
 198     name => {
 199         type => 'string',
 200         pattern => '[a-zA-Z0-9\-]+',
 201         optional => 1,
 202         format_description => 'string',
 203         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 204     },
 205 };
 206
 207 my $audio_fmt = {
 208     device => {
 209         type => 'string',
 210         enum => [qw(ich9-intel-hda intel-hda AC97)],
 211         description =>  "Configure an audio device."
 212     },
 213     driver =>  {
 214         type => 'string',
 215         enum => ['spice', 'none'],
 216         default => 'spice',
 217         optional => 1,
 218         description => "Driver backend for the audio device."
 219     },
 220 };
 221
 222 my $spice_enhancements_fmt = {
 223     foldersharing => {
 224         type => 'boolean',
 225         optional => 1,
 226         default => '0',
 227         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 228     },
 229     videostreaming =>  {
 230         type => 'string',
 231         enum => ['off', 'all', 'filter'],
 232         default => 'off',
 233         optional => 1,
 234         description => "Enable video streaming. Uses compression for detected video streams."
 235     },
 236 };
 237
 238 my $rng_fmt = {
 239     source => {
 240         type => 'string',
 241         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 242         default_key => 1,
 243         description => "The file on the host to gather entropy from. In most"
 244                      . " cases /dev/urandom should be preferred over /dev/random"
 245                      . " to avoid entropy-starvation issues on the host. Using"
 246                      . " urandom does *not* decrease security in any meaningful"
 247                      . " way, as it's still seeded from real entropy, and the"
 248                      . " bytes provided will most likely be mixed with real"
 249                      . " entropy on the guest as well. /dev/hwrng can be used"
 250                      . " to pass through a hardware RNG from the host.",
 251     },
 252     max_bytes => {
 253         type => 'integer',
 254         description => "Maximum bytes of entropy injected into the guest every"
 255                      . " 'period' milliseconds. Prefer a lower value when using"
 256                      . " /dev/random as source. Use 0 to disable limiting"
 257                      . " (potentially dangerous!).",
 258         optional => 1,
 259
 260         # default is 1 KiB/s, provides enough entropy to the guest to avoid
 261         # boot-starvation issues (e.g. systemd etc...) while allowing no chance
 262         # of overwhelming the host, provided we're reading from /dev/urandom
 263         default => 1024,
 264     },
 265     period => {
 266         type => 'integer',
 267         description => "Every 'period' milliseconds the entropy-injection quota"
 268                      . " is reset, allowing the guest to retrieve another"
 269                      . " 'max_bytes' of entropy.",
 270         optional => 1,
 271         default => 1000,
 272     },
 273 };
 274
 275 my $confdesc = {
 276     onboot => {
 277         optional => 1,
 278         type => 'boolean',
 279         description => "Specifies whether a VM will be started during system bootup.",
 280         default => 0,
 281     },
 282     autostart => {
 283         optional => 1,
 284         type => 'boolean',
 285         description => "Automatic restart after crash (currently ignored).",
 286         default => 0,
 287     },
 288     hotplug => {
 289         optional => 1,
 290         type => 'string', format => 'pve-hotplug-features',
 291         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 292         default => 'network,disk,usb',
 293     },
 294     reboot => {
 295         optional => 1,
 296         type => 'boolean',
 297         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 298         default => 1,
 299     },
 300     lock => {
 301         optional => 1,
 302         type => 'string',
 303         description => "Lock/unlock the VM.",
 304         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 305     },
 306     cpulimit => {
 307         optional => 1,
 308         type => 'number',
 309         description => "Limit of CPU usage.",
 310         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 311         minimum => 0,
 312         maximum => 128,
 313         default => 0,
 314     },
 315     cpuunits => {
 316         optional => 1,
 317         type => 'integer',
 318         description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
 319         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
 320             ." The larger the number is, the more CPU time this VM gets. Number is relative to"
 321             ." weights of all the other running VMs.",
 322         minimum => 2,
 323         maximum => 262144,
 324         default => 'cgroup v1: 1024, cgroup v2: 100',
 325     },
 326     memory => {
 327         optional => 1,
 328         type => 'integer',
 329         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 330         minimum => 16,
 331         default => 512,
 332     },
 333     balloon => {
 334         optional => 1,
 335         type => 'integer',
 336         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 337         minimum => 0,
 338     },
 339     shares => {
 340         optional => 1,
 341         type => 'integer',
 342         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 343         minimum => 0,
 344         maximum => 50000,
 345         default => 1000,
 346     },
 347     keyboard => {
 348         optional => 1,
 349         type => 'string',
 350         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 351                        "It should not be necessary to set it.",
 352         enum => PVE::Tools::kvmkeymaplist(),
 353         default => undef,
 354     },
 355     name => {
 356         optional => 1,
 357         type => 'string', format => 'dns-name',
 358         description => "Set a name for the VM. Only used on the configuration web interface.",
 359     },
 360     scsihw => {
 361         optional => 1,
 362         type => 'string',
 363         description => "SCSI controller model",
 364         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 365         default => 'lsi',
 366     },
 367     description => {
 368         optional => 1,
 369         type => 'string',
 370         description => "Description for the VM. Shown in the web-interface VM's summary."
 371             ." This is saved as comment inside the configuration file.",
 372         maxLength => 1024 * 8,
 373     },
 374     ostype => {
 375         optional => 1,
 376         type => 'string',
 377         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 378         description => "Specify guest operating system.",
 379         verbose_description => <<EODESC,
 380 Specify guest operating system. This is used to enable special
 381 optimization/features for specific operating systems:
 382
 383 [horizontal]
 384 other;; unspecified OS
 385 wxp;; Microsoft Windows XP
 386 w2k;; Microsoft Windows 2000
 387 w2k3;; Microsoft Windows 2003
 388 w2k8;; Microsoft Windows 2008
 389 wvista;; Microsoft Windows Vista
 390 win7;; Microsoft Windows 7
 391 win8;; Microsoft Windows 8/2012/2012r2
 392 win10;; Microsoft Windows 10/2016/2019
 393 l24;; Linux 2.4 Kernel
 394 l26;; Linux 2.6 - 5.X Kernel
 395 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 396 EODESC
 397     },
 398     boot => {
 399         optional => 1,
 400         type => 'string', format => 'pve-qm-boot',
 401         description => "Specify guest boot order. Use with 'order=', usage with"
 402                      . " no key or 'legacy=' is deprecated.",
 403     },
 404     bootdisk => {
 405         optional => 1,
 406         type => 'string', format => 'pve-qm-bootdisk',
 407         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 408         pattern => '(ide|sata|scsi|virtio)\d+',
 409     },
 410     smp => {
 411         optional => 1,
 412         type => 'integer',
 413         description => "The number of CPUs. Please use option -sockets instead.",
 414         minimum => 1,
 415         default => 1,
 416     },
 417     sockets => {
 418         optional => 1,
 419         type => 'integer',
 420         description => "The number of CPU sockets.",
 421         minimum => 1,
 422         default => 1,
 423     },
 424     cores => {
 425         optional => 1,
 426         type => 'integer',
 427         description => "The number of cores per socket.",
 428         minimum => 1,
 429         default => 1,
 430     },
 431     numa => {
 432         optional => 1,
 433         type => 'boolean',
 434         description => "Enable/disable NUMA.",
 435         default => 0,
 436     },
 437     hugepages => {
 438         optional => 1,
 439         type => 'string',
 440         description => "Enable/disable hugepages memory.",
 441         enum => [qw(any 2 1024)],
 442     },
 443     keephugepages => {
 444         optional => 1,
 445         type => 'boolean',
 446         default => 0,
 447         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 448             ." after VM shutdown and can be used for subsequent starts.",
 449     },
 450     vcpus => {
 451         optional => 1,
 452         type => 'integer',
 453         description => "Number of hotplugged vcpus.",
 454         minimum => 1,
 455         default => 0,
 456     },
 457     acpi => {
 458         optional => 1,
 459         type => 'boolean',
 460         description => "Enable/disable ACPI.",
 461         default => 1,
 462     },
 463     agent => {
 464         optional => 1,
 465         description => "Enable/disable Qemu GuestAgent and its properties.",
 466         type => 'string',
 467         format => $agent_fmt,
 468     },
 469     kvm => {
 470         optional => 1,
 471         type => 'boolean',
 472         description => "Enable/disable KVM hardware virtualization.",
 473         default => 1,
 474     },
 475     tdf => {
 476         optional => 1,
 477         type => 'boolean',
 478         description => "Enable/disable time drift fix.",
 479         default => 0,
 480     },
 481     localtime => {
 482         optional => 1,
 483         type => 'boolean',
 484         description => "Set the real time clock to local time. This is enabled by default if ostype"
 485             ." indicates a Microsoft OS.",
 486     },
 487     freeze => {
 488         optional => 1,
 489         type => 'boolean',
 490         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 491     },
 492     vga => {
 493         optional => 1,
 494         type => 'string', format => $vga_fmt,
 495         description => "Configure the VGA hardware.",
 496         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 497             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 498             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 499             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 500             ." display server. For win* OS you can select how many independent displays you want,"
 501             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 502             ." using a serial device as terminal.",
 503     },
 504     watchdog => {
 505         optional => 1,
 506         type => 'string', format => 'pve-qm-watchdog',
 507         description => "Create a virtual hardware watchdog device.",
 508         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 509             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 510             ." else the watchdog will reset the guest (or execute the respective action specified)",
 511     },
 512     startdate => {
 513         optional => 1,
 514         type => 'string',
 515         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 516         description => "Set the initial date of the real time clock. Valid format for date are:"
 517             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 518         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 519         default => 'now',
 520     },
 521     startup =>  get_standard_option('pve-startup-order'),
 522     template => {
 523         optional => 1,
 524         type => 'boolean',
 525         description => "Enable/disable Template.",
 526         default => 0,
 527     },
 528     args => {
 529         optional => 1,
 530         type => 'string',
 531         description => "Arbitrary arguments passed to kvm.",
 532         verbose_description => <<EODESCR,
 533 Arbitrary arguments passed to kvm, for example:
 534
 535 args: -no-reboot -no-hpet
 536
 537 NOTE: this option is for experts only.
 538 EODESCR
 539     },
 540     tablet => {
 541         optional => 1,
 542         type => 'boolean',
 543         default => 1,
 544         description => "Enable/disable the USB tablet device.",
 545         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 546             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 547             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 548             ." may consider disabling this to save some context switches. This is turned off by"
 549             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 550     },
 551     migrate_speed => {
 552         optional => 1,
 553         type => 'integer',
 554         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 555         minimum => 0,
 556         default => 0,
 557     },
 558     migrate_downtime => {
 559         optional => 1,
 560         type => 'number',
 561         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 562         minimum => 0,
 563         default => 0.1,
 564     },
 565     cdrom => {
 566         optional => 1,
 567         type => 'string', format => 'pve-qm-ide',
 568         typetext => '<volume>',
 569         description => "This is an alias for option -ide2",
 570     },
 571     cpu => {
 572         optional => 1,
 573         description => "Emulated CPU type.",
 574         type => 'string',
 575         format => 'pve-vm-cpu-conf',
 576     },
 577     parent => get_standard_option('pve-snapshot-name', {
 578         optional => 1,
 579         description => "Parent snapshot name. This is used internally, and should not be modified.",
 580     }),
 581     snaptime => {
 582         optional => 1,
 583         description => "Timestamp for snapshots.",
 584         type => 'integer',
 585         minimum => 0,
 586     },
 587     vmstate => {
 588         optional => 1,
 589         type => 'string', format => 'pve-volume-id',
 590         description => "Reference to a volume which stores the VM state. This is used internally"
 591             ." for snapshots.",
 592     },
 593     vmstatestorage => get_standard_option('pve-storage-id', {
 594         description => "Default storage for VM state volumes/files.",
 595         optional => 1,
 596     }),
 597     runningmachine => get_standard_option('pve-qemu-machine', {
 598         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 599             ." for snapshots.",
 600     }),
 601     runningcpu => {
 602         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 603             ." internally for snapshots.",
 604         optional => 1,
 605         type => 'string',
 606         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 607         format_description => 'QEMU -cpu parameter'
 608     },
 609     machine => get_standard_option('pve-qemu-machine'),
 610     arch => {
 611         description => "Virtual processor architecture. Defaults to the host.",
 612         optional => 1,
 613         type => 'string',
 614         enum => [qw(x86_64 aarch64)],
 615     },
 616     smbios1 => {
 617         description => "Specify SMBIOS type 1 fields.",
 618         type => 'string', format => 'pve-qm-smbios1',
 619         maxLength => 512,
 620         optional => 1,
 621     },
 622     protection => {
 623         optional => 1,
 624         type => 'boolean',
 625         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 626             ." remove disk operations.",
 627         default => 0,
 628     },
 629     bios => {
 630         optional => 1,
 631         type => 'string',
 632         enum => [ qw(seabios ovmf) ],
 633         description => "Select BIOS implementation.",
 634         default => 'seabios',
 635     },
 636     vmgenid => {
 637         type => 'string',
 638         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 639         format_description => 'UUID',
 640         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 641             ." to disable explicitly.",
 642         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 643             ." value identifier to the guest OS. This allows to notify the guest operating system"
 644             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 645             ." execution or creation from a template). The guest operating system notices the"
 646             ." change, and is then able to react as appropriate by marking its copies of"
 647             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 648             ."Note that auto-creation only works when done through API/CLI create or update methods"
 649             .", but not when manually editing the config file.",
 650         default => "1 (autogenerated)",
 651         optional => 1,
 652     },
 653     hookscript => {
 654         type => 'string',
 655         format => 'pve-volume-id',
 656         optional => 1,
 657         description => "Script that will be executed during various steps in the vms lifetime.",
 658     },
 659     ivshmem => {
 660         type => 'string',
 661         format => $ivshmem_fmt,
 662         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 663             ." the host.",
 664         optional => 1,
 665     },
 666     audio0 => {
 667         type => 'string',
 668         format => $audio_fmt,
 669         description => "Configure a audio device, useful in combination with QXL/Spice.",
 670         optional => 1
 671     },
 672     spice_enhancements => {
 673         type => 'string',
 674         format => $spice_enhancements_fmt,
 675         description => "Configure additional enhancements for SPICE.",
 676         optional => 1
 677     },
 678     tags => {
 679         type => 'string', format => 'pve-tag-list',
 680         description => 'Tags of the VM. This is only meta information.',
 681         optional => 1,
 682     },
 683     rng0 => {
 684         type => 'string',
 685         format => $rng_fmt,
 686         description => "Configure a VirtIO-based Random Number Generator.",
 687         optional => 1,
 688     },
 689 };
 690
 691 my $cicustom_fmt = {
 692     meta => {
 693         type => 'string',
 694         optional => 1,
 695         description => 'Specify a custom file containing all meta data passed to the VM via"
 696             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 697         format => 'pve-volume-id',
 698         format_description => 'volume',
 699     },
 700     network => {
 701         type => 'string',
 702         optional => 1,
 703         description => 'Specify a custom file containing all network data passed to the VM via'
 704             .' cloud-init.',
 705         format => 'pve-volume-id',
 706         format_description => 'volume',
 707     },
 708     user => {
 709         type => 'string',
 710         optional => 1,
 711         description => 'Specify a custom file containing all user data passed to the VM via'
 712             .' cloud-init.',
 713         format => 'pve-volume-id',
 714         format_description => 'volume',
 715     },
 716 };
 717 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 718
 719 my $confdesc_cloudinit = {
 720     citype => {
 721         optional => 1,
 722         type => 'string',
 723         description => 'Specifies the cloud-init configuration format. The default depends on the'
 724             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 725             .' and `configdrive2` for windows.',
 726         enum => ['configdrive2', 'nocloud', 'opennebula'],
 727     },
 728     ciuser => {
 729         optional => 1,
 730         type => 'string',
 731         description => "cloud-init: User name to change ssh keys and password for instead of the"
 732             ." image's configured default user.",
 733     },
 734     cipassword => {
 735         optional => 1,
 736         type => 'string',
 737         description => 'cloud-init: Password to assign the user. Using this is generally not'
 738             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 739             .' support hashed passwords.',
 740     },
 741     cicustom => {
 742         optional => 1,
 743         type => 'string',
 744         description => 'cloud-init: Specify custom files to replace the automatically generated'
 745             .' ones at start.',
 746         format => 'pve-qm-cicustom',
 747     },
 748     searchdomain => {
 749         optional => 1,
 750         type => 'string',
 751         description => "cloud-init: Sets DNS search domains for a container. Create will'
 752             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 753             .' are set.",
 754     },
 755     nameserver => {
 756         optional => 1,
 757         type => 'string', format => 'address-list',
 758         description => "cloud-init: Sets DNS server IP address for a container. Create will'
 759             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 760             .' are set.",
 761     },
 762     sshkeys => {
 763         optional => 1,
 764         type => 'string',
 765         format => 'urlencoded',
 766         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 767     },
 768 };
 769
 770 # what about other qemu settings ?
 771 #cpu => 'string',
 772 #machine => 'string',
 773 #fda => 'file',
 774 #fdb => 'file',
 775 #mtdblock => 'file',
 776 #sd => 'file',
 777 #pflash => 'file',
 778 #snapshot => 'bool',
 779 #bootp => 'file',
 780 ##tftp => 'dir',
 781 ##smb => 'dir',
 782 #kernel => 'file',
 783 #append => 'string',
 784 #initrd => 'file',
 785 ##soundhw => 'string',
 786
 787 while (my ($k, $v) = each %$confdesc) {
 788     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 789 }
 790
 791 my $MAX_USB_DEVICES = 5;
 792 my $MAX_NETS = 32;
 793 my $MAX_SERIAL_PORTS = 4;
 794 my $MAX_PARALLEL_PORTS = 3;
 795 my $MAX_NUMA = 8;
 796
 797 my $numa_fmt = {
 798     cpus => {
 799         type => "string",
 800         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 801         description => "CPUs accessing this NUMA node.",
 802         format_description => "id[-id];...",
 803     },
 804     memory => {
 805         type => "number",
 806         description => "Amount of memory this NUMA node provides.",
 807         optional => 1,
 808     },
 809     hostnodes => {
 810         type => "string",
 811         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 812         description => "Host NUMA nodes to use.",
 813         format_description => "id[-id];...",
 814         optional => 1,
 815     },
 816     policy => {
 817         type => 'string',
 818         enum => [qw(preferred bind interleave)],
 819         description => "NUMA allocation policy.",
 820         optional => 1,
 821     },
 822 };
 823 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 824 my $numadesc = {
 825     optional => 1,
 826     type => 'string', format => $numa_fmt,
 827     description => "NUMA topology.",
 828 };
 829 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 830
 831 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 832     $confdesc->{"numa$i"} = $numadesc;
 833 }
 834
 835 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 836                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 837                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 838 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 839
 840 my $net_fmt_bridge_descr = <<__EOD__;
 841 Bridge to attach the network device to. The Proxmox VE standard bridge
 842 is called 'vmbr0'.
 843
 844 If you do not specify a bridge, we create a kvm user (NATed) network
 845 device, which provides DHCP and DNS services. The following addresses
 846 are used:
 847
 848  10.0.2.2   Gateway
 849  10.0.2.3   DNS Server
 850  10.0.2.4   SMB Server
 851
 852 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 853 __EOD__
 854
 855 my $net_fmt = {
 856     macaddr  => get_standard_option('mac-addr', {
 857         description => "MAC address. That address must be unique withing your network. This is"
 858             ." automatically generated if not specified.",
 859     }),
 860     model => {
 861         type => 'string',
 862         description => "Network Card Model. The 'virtio' model provides the best performance with"
 863             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 864             ." best to use 'e1000'.",
 865         enum => $nic_model_list,
 866         default_key => 1,
 867     },
 868     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 869     bridge => {
 870         type => 'string',
 871         description => $net_fmt_bridge_descr,
 872         format_description => 'bridge',
 873         pattern => '[-_.\w\d]+',
 874         optional => 1,
 875     },
 876     queues => {
 877         type => 'integer',
 878         minimum => 0, maximum => 16,
 879         description => 'Number of packet queues to be used on the device.',
 880         optional => 1,
 881     },
 882     rate => {
 883         type => 'number',
 884         minimum => 0,
 885         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 886         optional => 1,
 887     },
 888     tag => {
 889         type => 'integer',
 890         minimum => 1, maximum => 4094,
 891         description => 'VLAN tag to apply to packets on this interface.',
 892         optional => 1,
 893     },
 894     trunks => {
 895         type => 'string',
 896         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 897         description => 'VLAN trunks to pass through this interface.',
 898         format_description => 'vlanid[;vlanid...]',
 899         optional => 1,
 900     },
 901     firewall => {
 902         type => 'boolean',
 903         description => 'Whether this interface should be protected by the firewall.',
 904         optional => 1,
 905     },
 906     link_down => {
 907         type => 'boolean',
 908         description => 'Whether this interface should be disconnected (like pulling the plug).',
 909         optional => 1,
 910     },
 911     mtu => {
 912         type => 'integer',
 913         minimum => 1, maximum => 65520,
 914         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 915         optional => 1,
 916     },
 917 };
 918
 919 my $netdesc = {
 920     optional => 1,
 921     type => 'string', format => $net_fmt,
 922     description => "Specify network devices.",
 923 };
 924
 925 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 926
 927 my $ipconfig_fmt = {
 928     ip => {
 929         type => 'string',
 930         format => 'pve-ipv4-config',
 931         format_description => 'IPv4Format/CIDR',
 932         description => 'IPv4 address in CIDR format.',
 933         optional => 1,
 934         default => 'dhcp',
 935     },
 936     gw => {
 937         type => 'string',
 938         format => 'ipv4',
 939         format_description => 'GatewayIPv4',
 940         description => 'Default gateway for IPv4 traffic.',
 941         optional => 1,
 942         requires => 'ip',
 943     },
 944     ip6 => {
 945         type => 'string',
 946         format => 'pve-ipv6-config',
 947         format_description => 'IPv6Format/CIDR',
 948         description => 'IPv6 address in CIDR format.',
 949         optional => 1,
 950         default => 'dhcp',
 951     },
 952     gw6 => {
 953         type => 'string',
 954         format => 'ipv6',
 955         format_description => 'GatewayIPv6',
 956         description => 'Default gateway for IPv6 traffic.',
 957         optional => 1,
 958         requires => 'ip6',
 959     },
 960 };
 961 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 962 my $ipconfigdesc = {
 963     optional => 1,
 964     type => 'string', format => 'pve-qm-ipconfig',
 965     description => <<'EODESCR',
 966 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 967
 968 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 969
 970 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
 971 gateway should be provided.
 972 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
 973 cloud-init 19.4 or newer.
 974
 975 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
 976 dhcp on IPv4.
 977 EODESCR
 978 };
 979 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 980
 981 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 982     $confdesc->{"net$i"} = $netdesc;
 983     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 984 }
 985
 986 foreach my $key (keys %$confdesc_cloudinit) {
 987     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 988 }
 989
 990 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 991 sub verify_volume_id_or_qm_path {
 992     my ($volid, $noerr) = @_;
 993
 994     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 995         return $volid;
 996     }
 997
 998     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 999     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1000     if ($@) {
1001         return if $noerr;
1002         die $@;
1003     }
1004     return $volid;
1005 }
1006
1007 my $usb_fmt = {
1008     host => {
1009         default_key => 1,
1010         type => 'string', format => 'pve-qm-usb-device',
1011         format_description => 'HOSTUSBDEVICE|spice',
1012         description => <<EODESCR,
1013 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1014
1015  'bus-port(.port)*' (decimal numbers) or
1016  'vendor_id:product_id' (hexadeciaml numbers) or
1017  'spice'
1018
1019 You can use the 'lsusb -t' command to list existing usb devices.
1020
1021 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1022 machines - use with special care.
1023
1024 The value 'spice' can be used to add a usb redirection devices for spice.
1025 EODESCR
1026     },
1027     usb3 => {
1028         optional => 1,
1029         type => 'boolean',
1030         description => "Specifies whether if given host option is a USB3 device or port.",
1031         default => 0,
1032     },
1033 };
1034
1035 my $usbdesc = {
1036     optional => 1,
1037     type => 'string', format => $usb_fmt,
1038     description => "Configure an USB device (n is 0 to 4).",
1039 };
1040 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1041
1042 my $serialdesc = {
1043         optional => 1,
1044         type => 'string',
1045         pattern => '(/dev/.+|socket)',
1046         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1047         verbose_description =>  <<EODESCR,
1048 Create a serial device inside the VM (n is 0 to 3), and pass through a
1049 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1050 host side (use 'qm terminal' to open a terminal connection).
1051
1052 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1053 use with special care.
1054
1055 CAUTION: Experimental! User reported problems with this option.
1056 EODESCR
1057 };
1058
1059 my $paralleldesc= {
1060         optional => 1,
1061         type => 'string',
1062         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1063         description =>  "Map host parallel devices (n is 0 to 2).",
1064         verbose_description =>  <<EODESCR,
1065 Map host parallel devices (n is 0 to 2).
1066
1067 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1068 machines - use with special care.
1069
1070 CAUTION: Experimental! User reported problems with this option.
1071 EODESCR
1072 };
1073
1074 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1075     $confdesc->{"parallel$i"} = $paralleldesc;
1076 }
1077
1078 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1079     $confdesc->{"serial$i"} = $serialdesc;
1080 }
1081
1082 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1083     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1084 }
1085
1086 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1087     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1088 }
1089
1090 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1091     $confdesc->{"usb$i"} = $usbdesc;
1092 }
1093
1094 my $boot_fmt = {
1095     legacy => {
1096         optional => 1,
1097         default_key => 1,
1098         type => 'string',
1099         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1100                      . " Deprecated, use 'order=' instead.",
1101         pattern => '[acdn]{1,4}',
1102         format_description => "[acdn]{1,4}",
1103
1104         # note: this is also the fallback if boot: is not given at all
1105         default => 'cdn',
1106     },
1107     order => {
1108         optional => 1,
1109         type => 'string',
1110         format => 'pve-qm-bootdev-list',
1111         format_description => "device[;device...]",
1112         description => <<EODESC,
1113 The guest will attempt to boot from devices in the order they appear here.
1114
1115 Disks, optical drives and passed-through storage USB devices will be directly
1116 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1117 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1118
1119 Note that only devices in this list will be marked as bootable and thus loaded
1120 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1121 (e.g. software-raid), you need to specify all of them here.
1122
1123 Overrides the deprecated 'legacy=[acdn]*' value when given.
1124 EODESC
1125     },
1126 };
1127 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1128
1129 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1130 sub verify_bootdev {
1131     my ($dev, $noerr) = @_;
1132
1133     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && $dev !~ m/^efidisk/;
1134
1135     my $check = sub {
1136         my ($base) = @_;
1137         return 0 if $dev !~ m/^$base\d+$/;
1138         return 0 if !$confdesc->{$dev};
1139         return 1;
1140     };
1141
1142     return $dev if $check->("net");
1143     return $dev if $check->("usb");
1144     return $dev if $check->("hostpci");
1145
1146     return if $noerr;
1147     die "invalid boot device '$dev'\n";
1148 }
1149
1150 sub print_bootorder {
1151     my ($devs) = @_;
1152     return "" if !@$devs;
1153     my $data = { order => join(';', @$devs) };
1154     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1155 }
1156
1157 my $kvm_api_version = 0;
1158
1159 sub kvm_version {
1160     return $kvm_api_version if $kvm_api_version;
1161
1162     open my $fh, '<', '/dev/kvm' or return;
1163
1164     # 0xae00 => KVM_GET_API_VERSION
1165     $kvm_api_version = ioctl($fh, 0xae00, 0);
1166     close($fh);
1167
1168     return $kvm_api_version;
1169 }
1170
1171 my $kvm_user_version = {};
1172 my $kvm_mtime = {};
1173
1174 sub kvm_user_version {
1175     my ($binary) = @_;
1176
1177     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1178     my $st = stat($binary);
1179
1180     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1181     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1182         $cachedmtime == $st->mtime;
1183
1184     $kvm_user_version->{$binary} = 'unknown';
1185     $kvm_mtime->{$binary} = $st->mtime;
1186
1187     my $code = sub {
1188         my $line = shift;
1189         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1190             $kvm_user_version->{$binary} = $2;
1191         }
1192     };
1193
1194     eval { run_command([$binary, '--version'], outfunc => $code); };
1195     warn $@ if $@;
1196
1197     return $kvm_user_version->{$binary};
1198
1199 }
1200 my sub extract_version {
1201     my ($machine_type, $version) = @_;
1202     $version = kvm_user_version() if !defined($version);
1203     PVE::QemuServer::Machine::extract_version($machine_type, $version)
1204 }
1205
1206 sub kernel_has_vhost_net {
1207     return -c '/dev/vhost-net';
1208 }
1209
1210 sub option_exists {
1211     my $key = shift;
1212     return defined($confdesc->{$key});
1213 }
1214
1215 my $cdrom_path;
1216 sub get_cdrom_path {
1217
1218     return  $cdrom_path if $cdrom_path;
1219
1220     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1221     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1222     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1223 }
1224
1225 sub get_iso_path {
1226     my ($storecfg, $vmid, $cdrom) = @_;
1227
1228     if ($cdrom eq 'cdrom') {
1229         return get_cdrom_path();
1230     } elsif ($cdrom eq 'none') {
1231         return '';
1232     } elsif ($cdrom =~ m|^/|) {
1233         return $cdrom;
1234     } else {
1235         return PVE::Storage::path($storecfg, $cdrom);
1236     }
1237 }
1238
1239 # try to convert old style file names to volume IDs
1240 sub filename_to_volume_id {
1241     my ($vmid, $file, $media) = @_;
1242
1243      if (!($file eq 'none' || $file eq 'cdrom' ||
1244           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1245
1246         return if $file =~ m|/|;
1247
1248         if ($media && $media eq 'cdrom') {
1249             $file = "local:iso/$file";
1250         } else {
1251             $file = "local:$vmid/$file";
1252         }
1253     }
1254
1255     return $file;
1256 }
1257
1258 sub verify_media_type {
1259     my ($opt, $vtype, $media) = @_;
1260
1261     return if !$media;
1262
1263     my $etype;
1264     if ($media eq 'disk') {
1265         $etype = 'images';
1266     } elsif ($media eq 'cdrom') {
1267         $etype = 'iso';
1268     } else {
1269         die "internal error";
1270     }
1271
1272     return if ($vtype eq $etype);
1273
1274     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1275 }
1276
1277 sub cleanup_drive_path {
1278     my ($opt, $storecfg, $drive) = @_;
1279
1280     # try to convert filesystem paths to volume IDs
1281
1282     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1283         ($drive->{file} !~ m|^/dev/.+|) &&
1284         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1285         ($drive->{file} !~ m/^\d+$/)) {
1286         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1287         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1288             if !$vtype;
1289         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1290         verify_media_type($opt, $vtype, $drive->{media});
1291         $drive->{file} = $volid;
1292     }
1293
1294     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1295 }
1296
1297 sub parse_hotplug_features {
1298     my ($data) = @_;
1299
1300     my $res = {};
1301
1302     return $res if $data eq '0';
1303
1304     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1305
1306     foreach my $feature (PVE::Tools::split_list($data)) {
1307         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1308             $res->{$1} = 1;
1309         } else {
1310             die "invalid hotplug feature '$feature'\n";
1311         }
1312     }
1313     return $res;
1314 }
1315
1316 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1317 sub pve_verify_hotplug_features {
1318     my ($value, $noerr) = @_;
1319
1320     return $value if parse_hotplug_features($value);
1321
1322     return if $noerr;
1323
1324     die "unable to parse hotplug option\n";
1325 }
1326
1327 sub scsi_inquiry {
1328     my($fh, $noerr) = @_;
1329
1330     my $SG_IO = 0x2285;
1331     my $SG_GET_VERSION_NUM = 0x2282;
1332
1333     my $versionbuf = "\x00" x 8;
1334     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1335     if (!$ret) {
1336         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1337         return;
1338     }
1339     my $version = unpack("I", $versionbuf);
1340     if ($version < 30000) {
1341         die "scsi generic interface too old\n"  if !$noerr;
1342         return;
1343     }
1344
1345     my $buf = "\x00" x 36;
1346     my $sensebuf = "\x00" x 8;
1347     my $cmd = pack("C x3 C x1", 0x12, 36);
1348
1349     # see /usr/include/scsi/sg.h
1350     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1351
1352     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1353                       length($sensebuf), 0, length($buf), $buf,
1354                       $cmd, $sensebuf, 6000);
1355
1356     $ret = ioctl($fh, $SG_IO, $packet);
1357     if (!$ret) {
1358         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1359         return;
1360     }
1361
1362     my @res = unpack($sg_io_hdr_t, $packet);
1363     if ($res[17] || $res[18]) {
1364         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1365         return;
1366     }
1367
1368     my $res = {};
1369     (my $byte0, my $byte1, $res->{vendor},
1370      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1371
1372     $res->{removable} = $byte1 & 128 ? 1 : 0;
1373     $res->{type} = $byte0 & 31;
1374
1375     return $res;
1376 }
1377
1378 sub path_is_scsi {
1379     my ($path) = @_;
1380
1381     my $fh = IO::File->new("+<$path") || return;
1382     my $res = scsi_inquiry($fh, 1);
1383     close($fh);
1384
1385     return $res;
1386 }
1387
1388 sub print_tabletdevice_full {
1389     my ($conf, $arch) = @_;
1390
1391     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1392
1393     # we use uhci for old VMs because tablet driver was buggy in older qemu
1394     my $usbbus;
1395     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1396         $usbbus = 'ehci';
1397     } else {
1398         $usbbus = 'uhci';
1399     }
1400
1401     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1402 }
1403
1404 sub print_keyboarddevice_full {
1405     my ($conf, $arch, $machine) = @_;
1406
1407     return if $arch ne 'aarch64';
1408
1409     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1410 }
1411
1412 my sub get_drive_id {
1413     my ($drive) = @_;
1414     return "$drive->{interface}$drive->{index}";
1415 }
1416
1417 sub print_drivedevice_full {
1418     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1419
1420     my $device = '';
1421     my $maxdev = 0;
1422
1423     my $drive_id = get_drive_id($drive);
1424     if ($drive->{interface} eq 'virtio') {
1425         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1426         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1427         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1428     } elsif ($drive->{interface} eq 'scsi') {
1429
1430         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1431         my $unit = $drive->{index} % $maxdev;
1432         my $devicetype = 'hd';
1433         my $path = '';
1434         if (drive_is_cdrom($drive)) {
1435             $devicetype = 'cd';
1436         } else {
1437             if ($drive->{file} =~ m|^/|) {
1438                 $path = $drive->{file};
1439                 if (my $info = path_is_scsi($path)) {
1440                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1441                         $devicetype = 'block';
1442                     } elsif ($info->{type} == 1) { # tape
1443                         $devicetype = 'generic';
1444                     }
1445                 }
1446             } else {
1447                  $path = PVE::Storage::path($storecfg, $drive->{file});
1448             }
1449
1450             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1451             my $version = extract_version($machine_type, kvm_user_version());
1452             if ($path =~ m/^iscsi\:\/\// &&
1453                !min_version($version, 4, 1)) {
1454                 $devicetype = 'generic';
1455             }
1456         }
1457
1458         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1459             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1460         } else {
1461             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1462                 .",lun=$drive->{index}";
1463         }
1464         $device .= ",drive=drive-$drive_id,id=$drive_id";
1465
1466         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1467             $device .= ",rotation_rate=1";
1468         }
1469         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1470
1471     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1472         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1473         my $controller = int($drive->{index} / $maxdev);
1474         my $unit = $drive->{index} % $maxdev;
1475         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1476
1477         $device = "ide-$devicetype";
1478         if ($drive->{interface} eq 'ide') {
1479             $device .= ",bus=ide.$controller,unit=$unit";
1480         } else {
1481             $device .= ",bus=ahci$controller.$unit";
1482         }
1483         $device .= ",drive=drive-$drive_id,id=$drive_id";
1484
1485         if ($devicetype eq 'hd') {
1486             if (my $model = $drive->{model}) {
1487                 $model = URI::Escape::uri_unescape($model);
1488                 $device .= ",model=$model";
1489             }
1490             if ($drive->{ssd}) {
1491                 $device .= ",rotation_rate=1";
1492             }
1493         }
1494         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1495     } elsif ($drive->{interface} eq 'usb') {
1496         die "implement me";
1497         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1498     } else {
1499         die "unsupported interface type";
1500     }
1501
1502     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1503
1504     if (my $serial = $drive->{serial}) {
1505         $serial = URI::Escape::uri_unescape($serial);
1506         $device .= ",serial=$serial";
1507     }
1508
1509
1510     return $device;
1511 }
1512
1513 sub get_initiator_name {
1514     my $initiator;
1515
1516     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1517     while (defined(my $line = <$fh>)) {
1518         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1519         $initiator = $1;
1520         last;
1521     }
1522     $fh->close();
1523
1524     return $initiator;
1525 }
1526
1527 sub print_drive_commandline_full {
1528     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1529
1530     my $path;
1531     my $volid = $drive->{file};
1532     my $format = $drive->{format};
1533     my $drive_id = get_drive_id($drive);
1534
1535     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1536     my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1537
1538     if (drive_is_cdrom($drive)) {
1539         $path = get_iso_path($storecfg, $vmid, $volid);
1540         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1541     } else {
1542         if ($storeid) {
1543             $path = PVE::Storage::path($storecfg, $volid);
1544             $format //= qemu_img_format($scfg, $volname);
1545         } else {
1546             $path = $volid;
1547             $format //= "raw";
1548         }
1549    }
1550
1551    my $is_rbd = $path =~ m/^rbd:/;
1552
1553     my $opts = '';
1554     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1555     foreach my $o (@qemu_drive_options) {
1556         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1557     }
1558
1559     # snapshot only accepts on|off
1560     if (defined($drive->{snapshot})) {
1561         my $v = $drive->{snapshot} ? 'on' : 'off';
1562         $opts .= ",snapshot=$v";
1563     }
1564
1565     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1566         my ($dir, $qmpname) = @$type;
1567         if (my $v = $drive->{"mbps$dir"}) {
1568             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1569         }
1570         if (my $v = $drive->{"mbps${dir}_max"}) {
1571             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1572         }
1573         if (my $v = $drive->{"bps${dir}_max_length"}) {
1574             $opts .= ",throttling.bps$qmpname-max-length=$v";
1575         }
1576         if (my $v = $drive->{"iops${dir}"}) {
1577             $opts .= ",throttling.iops$qmpname=$v";
1578         }
1579         if (my $v = $drive->{"iops${dir}_max"}) {
1580             $opts .= ",throttling.iops$qmpname-max=$v";
1581         }
1582         if (my $v = $drive->{"iops${dir}_max_length"}) {
1583             $opts .= ",throttling.iops$qmpname-max-length=$v";
1584         }
1585     }
1586
1587     if ($pbs_name) {
1588         $format = "rbd" if $is_rbd;
1589         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1590             if !$format;
1591         $opts .= ",format=alloc-track,file.driver=$format";
1592     } elsif ($format) {
1593         $opts .= ",format=$format";
1594     }
1595
1596     my $cache_direct = 0;
1597
1598     if (my $cache = $drive->{cache}) {
1599         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1600     } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1601         $opts .= ",cache=none";
1602         $cache_direct = 1;
1603     }
1604
1605     # io_uring with cache mode writeback or writethrough on krbd will hang...
1606     my $rbd_no_io_uring = $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1607
1608     if (!$drive->{aio}) {
1609         if ($io_uring && !$rbd_no_io_uring) {
1610             # io_uring supports all cache modes
1611             $opts .= ",aio=io_uring";
1612         } else {
1613             # aio native works only with O_DIRECT
1614             if($cache_direct) {
1615                 $opts .= ",aio=native";
1616             } else {
1617                 $opts .= ",aio=threads";
1618             }
1619         }
1620     }
1621
1622     if (!drive_is_cdrom($drive)) {
1623         my $detectzeroes;
1624         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1625             $detectzeroes = 'off';
1626         } elsif ($drive->{discard}) {
1627             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1628         } else {
1629             # This used to be our default with discard not being specified:
1630             $detectzeroes = 'on';
1631         }
1632
1633         # note: 'detect-zeroes' works per blockdev and we want it to persist
1634         # after the alloc-track is removed, so put it on 'file' directly
1635         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1636         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1637     }
1638
1639     if ($pbs_name) {
1640         $opts .= ",backing=$pbs_name";
1641         $opts .= ",auto-remove=on";
1642     }
1643
1644     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1645     my $file_param = "file";
1646     if ($pbs_name) {
1647         # non-rbd drivers require the underlying file to be a seperate block
1648         # node, so add a second .file indirection
1649         $file_param .= ".file" if !$is_rbd;
1650         $file_param .= ".filename";
1651     }
1652     my $pathinfo = $path ? "$file_param=$path," : '';
1653
1654     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1655 }
1656
1657 sub print_pbs_blockdev {
1658     my ($pbs_conf, $pbs_name) = @_;
1659     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1660     $blockdev .= ",repository=$pbs_conf->{repository}";
1661     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1662     $blockdev .= ",archive=$pbs_conf->{archive}";
1663     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1664     return $blockdev;
1665 }
1666
1667 sub print_netdevice_full {
1668     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1669
1670     my $device = $net->{model};
1671     if ($net->{model} eq 'virtio') {
1672          $device = 'virtio-net-pci';
1673      };
1674
1675     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1676     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1677     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1678         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1679         # and out of each queue plus one config interrupt and control vector queue
1680         my $vectors = $net->{queues} * 2 + 2;
1681         $tmpstr .= ",vectors=$vectors,mq=on";
1682     }
1683     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1684
1685     if (my $mtu = $net->{mtu}) {
1686         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1687             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1688             if ($mtu == 1) {
1689                  $mtu = $bridge_mtu;
1690             } elsif ($mtu < 576) {
1691                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1692             } elsif ($mtu > $bridge_mtu) {
1693                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1694             }
1695             $tmpstr .= ",host_mtu=$mtu";
1696         } else {
1697             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1698         }
1699     }
1700
1701     if ($use_old_bios_files) {
1702         my $romfile;
1703         if ($device eq 'virtio-net-pci') {
1704             $romfile = 'pxe-virtio.rom';
1705         } elsif ($device eq 'e1000') {
1706             $romfile = 'pxe-e1000.rom';
1707         } elsif ($device eq 'ne2k') {
1708             $romfile = 'pxe-ne2k_pci.rom';
1709         } elsif ($device eq 'pcnet') {
1710             $romfile = 'pxe-pcnet.rom';
1711         } elsif ($device eq 'rtl8139') {
1712             $romfile = 'pxe-rtl8139.rom';
1713         }
1714         $tmpstr .= ",romfile=$romfile" if $romfile;
1715     }
1716
1717     return $tmpstr;
1718 }
1719
1720 sub print_netdev_full {
1721     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1722
1723     my $i = '';
1724     if ($netid =~ m/^net(\d+)$/) {
1725         $i = int($1);
1726     }
1727
1728     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1729
1730     my $ifname = "tap${vmid}i$i";
1731
1732     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1733     die "interface name '$ifname' is too long (max 15 character)\n"
1734         if length($ifname) >= 16;
1735
1736     my $vhostparam = '';
1737     if (is_native($arch)) {
1738         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1739     }
1740
1741     my $vmname = $conf->{name} || "vm$vmid";
1742
1743     my $netdev = "";
1744     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1745
1746     if ($net->{bridge}) {
1747         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1748             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1749     } else {
1750         $netdev = "type=user,id=$netid,hostname=$vmname";
1751     }
1752
1753     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1754
1755     return $netdev;
1756 }
1757
1758 my $vga_map = {
1759     'cirrus' => 'cirrus-vga',
1760     'std' => 'VGA',
1761     'vmware' => 'vmware-svga',
1762     'virtio' => 'virtio-vga',
1763 };
1764
1765 sub print_vga_device {
1766     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1767
1768     my $type = $vga_map->{$vga->{type}};
1769     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1770         $type = 'virtio-gpu';
1771     }
1772     my $vgamem_mb = $vga->{memory};
1773
1774     my $max_outputs = '';
1775     if ($qxlnum) {
1776         $type = $id ? 'qxl' : 'qxl-vga';
1777
1778         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1779             # set max outputs so linux can have up to 4 qxl displays with one device
1780             if (min_version($machine_version, 4, 1)) {
1781                 $max_outputs = ",max_outputs=4";
1782             }
1783         }
1784     }
1785
1786     die "no devicetype for $vga->{type}\n" if !$type;
1787
1788     my $memory = "";
1789     if ($vgamem_mb) {
1790         if ($vga->{type} eq 'virtio') {
1791             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1792             $memory = ",max_hostmem=$bytes";
1793         } elsif ($qxlnum) {
1794             # from https://www.spice-space.org/multiple-monitors.html
1795             $memory = ",vgamem_mb=$vga->{memory}";
1796             my $ram = $vgamem_mb * 4;
1797             my $vram = $vgamem_mb * 2;
1798             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1799         } else {
1800             $memory = ",vgamem_mb=$vga->{memory}";
1801         }
1802     } elsif ($qxlnum && $id) {
1803         $memory = ",ram_size=67108864,vram_size=33554432";
1804     }
1805
1806     my $edidoff = "";
1807     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1808         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1809     }
1810
1811     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1812     my $vgaid = "vga" . ($id // '');
1813     my $pciaddr;
1814
1815     if ($q35 && $vgaid eq 'vga') {
1816         # the first display uses pcie.0 bus on q35 machines
1817         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1818     } else {
1819         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1820     }
1821
1822     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1823 }
1824
1825 sub parse_number_sets {
1826     my ($set) = @_;
1827     my $res = [];
1828     foreach my $part (split(/;/, $set)) {
1829         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1830             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1831             push @$res, [ $1, $2 ];
1832         } else {
1833             die "invalid range: $part\n";
1834         }
1835     }
1836     return $res;
1837 }
1838
1839 sub parse_numa {
1840     my ($data) = @_;
1841
1842     my $res = parse_property_string($numa_fmt, $data);
1843     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1844     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1845     return $res;
1846 }
1847
1848 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1849 sub parse_net {
1850     my ($data) = @_;
1851
1852     my $res = eval { parse_property_string($net_fmt, $data) };
1853     if ($@) {
1854         warn $@;
1855         return;
1856     }
1857     if (!defined($res->{macaddr})) {
1858         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1859         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1860     }
1861     return $res;
1862 }
1863
1864 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1865 sub parse_ipconfig {
1866     my ($data) = @_;
1867
1868     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1869     if ($@) {
1870         warn $@;
1871         return;
1872     }
1873
1874     if ($res->{gw} && !$res->{ip}) {
1875         warn 'gateway specified without specifying an IP address';
1876         return;
1877     }
1878     if ($res->{gw6} && !$res->{ip6}) {
1879         warn 'IPv6 gateway specified without specifying an IPv6 address';
1880         return;
1881     }
1882     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1883         warn 'gateway specified together with DHCP';
1884         return;
1885     }
1886     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1887         # gw6 + auto/dhcp
1888         warn "IPv6 gateway specified together with $res->{ip6} address";
1889         return;
1890     }
1891
1892     if (!$res->{ip} && !$res->{ip6}) {
1893         return { ip => 'dhcp', ip6 => 'dhcp' };
1894     }
1895
1896     return $res;
1897 }
1898
1899 sub print_net {
1900     my $net = shift;
1901
1902     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1903 }
1904
1905 sub add_random_macs {
1906     my ($settings) = @_;
1907
1908     foreach my $opt (keys %$settings) {
1909         next if $opt !~ m/^net(\d+)$/;
1910         my $net = parse_net($settings->{$opt});
1911         next if !$net;
1912         $settings->{$opt} = print_net($net);
1913     }
1914 }
1915
1916 sub vm_is_volid_owner {
1917     my ($storecfg, $vmid, $volid) = @_;
1918
1919     if ($volid !~  m|^/|) {
1920         my ($path, $owner);
1921         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1922         if ($owner && ($owner == $vmid)) {
1923             return 1;
1924         }
1925     }
1926
1927     return;
1928 }
1929
1930 sub vmconfig_register_unused_drive {
1931     my ($storecfg, $vmid, $conf, $drive) = @_;
1932
1933     if (drive_is_cloudinit($drive)) {
1934         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1935         warn $@ if $@;
1936     } elsif (!drive_is_cdrom($drive)) {
1937         my $volid = $drive->{file};
1938         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1939             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1940         }
1941     }
1942 }
1943
1944 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1945 my $smbios1_fmt = {
1946     uuid => {
1947         type => 'string',
1948         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1949         format_description => 'UUID',
1950         description => "Set SMBIOS1 UUID.",
1951         optional => 1,
1952     },
1953     version => {
1954         type => 'string',
1955         pattern => '[A-Za-z0-9+\/]+={0,2}',
1956         format_description => 'Base64 encoded string',
1957         description => "Set SMBIOS1 version.",
1958         optional => 1,
1959     },
1960     serial => {
1961         type => 'string',
1962         pattern => '[A-Za-z0-9+\/]+={0,2}',
1963         format_description => 'Base64 encoded string',
1964         description => "Set SMBIOS1 serial number.",
1965         optional => 1,
1966     },
1967     manufacturer => {
1968         type => 'string',
1969         pattern => '[A-Za-z0-9+\/]+={0,2}',
1970         format_description => 'Base64 encoded string',
1971         description => "Set SMBIOS1 manufacturer.",
1972         optional => 1,
1973     },
1974     product => {
1975         type => 'string',
1976         pattern => '[A-Za-z0-9+\/]+={0,2}',
1977         format_description => 'Base64 encoded string',
1978         description => "Set SMBIOS1 product ID.",
1979         optional => 1,
1980     },
1981     sku => {
1982         type => 'string',
1983         pattern => '[A-Za-z0-9+\/]+={0,2}',
1984         format_description => 'Base64 encoded string',
1985         description => "Set SMBIOS1 SKU string.",
1986         optional => 1,
1987     },
1988     family => {
1989         type => 'string',
1990         pattern => '[A-Za-z0-9+\/]+={0,2}',
1991         format_description => 'Base64 encoded string',
1992         description => "Set SMBIOS1 family string.",
1993         optional => 1,
1994     },
1995     base64 => {
1996         type => 'boolean',
1997         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
1998         optional => 1,
1999     },
2000 };
2001
2002 sub parse_smbios1 {
2003     my ($data) = @_;
2004
2005     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2006     warn $@ if $@;
2007     return $res;
2008 }
2009
2010 sub print_smbios1 {
2011     my ($smbios1) = @_;
2012     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2013 }
2014
2015 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2016
2017 sub parse_watchdog {
2018     my ($value) = @_;
2019
2020     return if !$value;
2021
2022     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2023     warn $@ if $@;
2024     return $res;
2025 }
2026
2027 sub parse_guest_agent {
2028     my ($conf) = @_;
2029
2030     return {} if !defined($conf->{agent});
2031
2032     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2033     warn $@ if $@;
2034
2035     # if the agent is disabled ignore the other potentially set properties
2036     return {} if !$res->{enabled};
2037     return $res;
2038 }
2039
2040 sub get_qga_key {
2041     my ($conf, $key) = @_;
2042     return undef if !defined($conf->{agent});
2043
2044     my $agent = parse_guest_agent($conf);
2045     return $agent->{$key};
2046 }
2047
2048 sub parse_vga {
2049     my ($value) = @_;
2050
2051     return {} if !$value;
2052     my $res = eval { parse_property_string($vga_fmt, $value) };
2053     warn $@ if $@;
2054     return $res;
2055 }
2056
2057 sub parse_rng {
2058     my ($value) = @_;
2059
2060     return if !$value;
2061
2062     my $res = eval { parse_property_string($rng_fmt, $value) };
2063     warn $@ if $@;
2064     return $res;
2065 }
2066
2067 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2068 sub verify_usb_device {
2069     my ($value, $noerr) = @_;
2070
2071     return $value if parse_usb_device($value);
2072
2073     return if $noerr;
2074
2075     die "unable to parse usb device\n";
2076 }
2077
2078 # add JSON properties for create and set function
2079 sub json_config_properties {
2080     my $prop = shift;
2081
2082     foreach my $opt (keys %$confdesc) {
2083         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' ||
2084             $opt eq 'runningmachine' || $opt eq 'runningcpu';
2085         $prop->{$opt} = $confdesc->{$opt};
2086     }
2087
2088     return $prop;
2089 }
2090
2091 # return copy of $confdesc_cloudinit to generate documentation
2092 sub cloudinit_config_properties {
2093
2094     return dclone($confdesc_cloudinit);
2095 }
2096
2097 sub check_type {
2098     my ($key, $value) = @_;
2099
2100     die "unknown setting '$key'\n" if !$confdesc->{$key};
2101
2102     my $type = $confdesc->{$key}->{type};
2103
2104     if (!defined($value)) {
2105         die "got undefined value\n";
2106     }
2107
2108     if ($value =~ m/[\n\r]/) {
2109         die "property contains a line feed\n";
2110     }
2111
2112     if ($type eq 'boolean') {
2113         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2114         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2115         die "type check ('boolean') failed - got '$value'\n";
2116     } elsif ($type eq 'integer') {
2117         return int($1) if $value =~ m/^(\d+)$/;
2118         die "type check ('integer') failed - got '$value'\n";
2119     } elsif ($type eq 'number') {
2120         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2121         die "type check ('number') failed - got '$value'\n";
2122     } elsif ($type eq 'string') {
2123         if (my $fmt = $confdesc->{$key}->{format}) {
2124             PVE::JSONSchema::check_format($fmt, $value);
2125             return $value;
2126         }
2127         $value =~ s/^\"(.*)\"$/$1/;
2128         return $value;
2129     } else {
2130         die "internal error"
2131     }
2132 }
2133
2134 sub destroy_vm {
2135     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2136
2137     my $conf = PVE::QemuConfig->load_config($vmid);
2138
2139     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2140
2141     if ($conf->{template}) {
2142         # check if any base image is still used by a linked clone
2143         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2144                 my ($ds, $drive) = @_;
2145                 return if drive_is_cdrom($drive);
2146
2147                 my $volid = $drive->{file};
2148                 return if !$volid || $volid =~ m|^/|;
2149
2150                 die "base volume '$volid' is still in use by linked cloned\n"
2151                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2152
2153         });
2154     }
2155
2156     my $remove_owned_drive = sub {
2157         my ($ds, $drive) = @_;
2158         return if drive_is_cdrom($drive, 1);
2159
2160         my $volid = $drive->{file};
2161         return if !$volid || $volid =~ m|^/|;
2162
2163         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2164         return if !$path || !$owner || ($owner != $vmid);
2165
2166         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2167         warn "Could not remove disk '$volid', check manually: $@" if $@;
2168     };
2169
2170     # only remove disks owned by this VM (referenced in the config)
2171     my $include_opts = {
2172         include_unused => 1,
2173         extra_keys => ['vmstate'],
2174     };
2175     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2176
2177     for my $snap (values %{$conf->{snapshots}}) {
2178         next if !defined($snap->{vmstate});
2179         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2180         next if !defined($drive);
2181         $remove_owned_drive->('vmstate', $drive);
2182     }
2183
2184     if ($purge_unreferenced) { # also remove unreferenced disk
2185         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2186         PVE::Storage::foreach_volid($vmdisks, sub {
2187             my ($volid, $sid, $volname, $d) = @_;
2188             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2189             warn $@ if $@;
2190         });
2191     }
2192
2193     if (defined $replacement_conf) {
2194         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2195     } else {
2196         PVE::QemuConfig->destroy_config($vmid);
2197     }
2198 }
2199
2200 sub parse_vm_config {
2201     my ($filename, $raw) = @_;
2202
2203     return if !defined($raw);
2204
2205     my $res = {
2206         digest => Digest::SHA::sha1_hex($raw),
2207         snapshots => {},
2208         pending => {},
2209     };
2210
2211     $filename =~ m|/qemu-server/(\d+)\.conf$|
2212         || die "got strange filename '$filename'";
2213
2214     my $vmid = $1;
2215
2216     my $conf = $res;
2217     my $descr;
2218     my $section = '';
2219
2220     my @lines = split(/\n/, $raw);
2221     foreach my $line (@lines) {
2222         next if $line =~ m/^\s*$/;
2223
2224         if ($line =~ m/^\[PENDING\]\s*$/i) {
2225             $section = 'pending';
2226             if (defined($descr)) {
2227                 $descr =~ s/\s+$//;
2228                 $conf->{description} = $descr;
2229             }
2230             $descr = undef;
2231             $conf = $res->{$section} = {};
2232             next;
2233
2234         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2235             $section = $1;
2236             if (defined($descr)) {
2237                 $descr =~ s/\s+$//;
2238                 $conf->{description} = $descr;
2239             }
2240             $descr = undef;
2241             $conf = $res->{snapshots}->{$section} = {};
2242             next;
2243         }
2244
2245         if ($line =~ m/^\#(.*)\s*$/) {
2246             $descr = '' if !defined($descr);
2247             $descr .= PVE::Tools::decode_text($1) . "\n";
2248             next;
2249         }
2250
2251         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2252             $descr = '' if !defined($descr);
2253             $descr .= PVE::Tools::decode_text($2);
2254         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2255             $conf->{snapstate} = $1;
2256         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2257             my $key = $1;
2258             my $value = $2;
2259             $conf->{$key} = $value;
2260         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2261             my $value = $1;
2262             if ($section eq 'pending') {
2263                 $conf->{delete} = $value; # we parse this later
2264             } else {
2265                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2266             }
2267         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2268             my $key = $1;
2269             my $value = $2;
2270             eval { $value = check_type($key, $value); };
2271             if ($@) {
2272                 warn "vm $vmid - unable to parse value of '$key' - $@";
2273             } else {
2274                 $key = 'ide2' if $key eq 'cdrom';
2275                 my $fmt = $confdesc->{$key}->{format};
2276                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2277                     my $v = parse_drive($key, $value);
2278                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2279                         $v->{file} = $volid;
2280                         $value = print_drive($v);
2281                     } else {
2282                         warn "vm $vmid - unable to parse value of '$key'\n";
2283                         next;
2284                     }
2285                 }
2286
2287                 $conf->{$key} = $value;
2288             }
2289         } else {
2290             warn "vm $vmid - unable to parse config: $line\n";
2291         }
2292     }
2293
2294     if (defined($descr)) {
2295         $descr =~ s/\s+$//;
2296         $conf->{description} = $descr;
2297     }
2298     delete $res->{snapstate}; # just to be sure
2299
2300     return $res;
2301 }
2302
2303 sub write_vm_config {
2304     my ($filename, $conf) = @_;
2305
2306     delete $conf->{snapstate}; # just to be sure
2307
2308     if ($conf->{cdrom}) {
2309         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2310         $conf->{ide2} = $conf->{cdrom};
2311         delete $conf->{cdrom};
2312     }
2313
2314     # we do not use 'smp' any longer
2315     if ($conf->{sockets}) {
2316         delete $conf->{smp};
2317     } elsif ($conf->{smp}) {
2318         $conf->{sockets} = $conf->{smp};
2319         delete $conf->{cores};
2320         delete $conf->{smp};
2321     }
2322
2323     my $used_volids = {};
2324
2325     my $cleanup_config = sub {
2326         my ($cref, $pending, $snapname) = @_;
2327
2328         foreach my $key (keys %$cref) {
2329             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2330                 $key eq 'snapstate' || $key eq 'pending';
2331             my $value = $cref->{$key};
2332             if ($key eq 'delete') {
2333                 die "propertry 'delete' is only allowed in [PENDING]\n"
2334                     if !$pending;
2335                 # fixme: check syntax?
2336                 next;
2337             }
2338             eval { $value = check_type($key, $value); };
2339             die "unable to parse value of '$key' - $@" if $@;
2340
2341             $cref->{$key} = $value;
2342
2343             if (!$snapname && is_valid_drivename($key)) {
2344                 my $drive = parse_drive($key, $value);
2345                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2346             }
2347         }
2348     };
2349
2350     &$cleanup_config($conf);
2351
2352     &$cleanup_config($conf->{pending}, 1);
2353
2354     foreach my $snapname (keys %{$conf->{snapshots}}) {
2355         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2356         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2357     }
2358
2359     # remove 'unusedX' settings if we re-add a volume
2360     foreach my $key (keys %$conf) {
2361         my $value = $conf->{$key};
2362         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2363             delete $conf->{$key};
2364         }
2365     }
2366
2367     my $generate_raw_config = sub {
2368         my ($conf, $pending) = @_;
2369
2370         my $raw = '';
2371
2372         # add description as comment to top of file
2373         if (defined(my $descr = $conf->{description})) {
2374             if ($descr) {
2375                 foreach my $cl (split(/\n/, $descr)) {
2376                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2377                 }
2378             } else {
2379                 $raw .= "#\n" if $pending;
2380             }
2381         }
2382
2383         foreach my $key (sort keys %$conf) {
2384             next if $key =~ /^(digest|description|pending|snapshots)$/;
2385             $raw .= "$key: $conf->{$key}\n";
2386         }
2387         return $raw;
2388     };
2389
2390     my $raw = &$generate_raw_config($conf);
2391
2392     if (scalar(keys %{$conf->{pending}})){
2393         $raw .= "\n[PENDING]\n";
2394         $raw .= &$generate_raw_config($conf->{pending}, 1);
2395     }
2396
2397     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2398         $raw .= "\n[$snapname]\n";
2399         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2400     }
2401
2402     return $raw;
2403 }
2404
2405 sub load_defaults {
2406
2407     my $res = {};
2408
2409     # we use static defaults from our JSON schema configuration
2410     foreach my $key (keys %$confdesc) {
2411         if (defined(my $default = $confdesc->{$key}->{default})) {
2412             $res->{$key} = $default;
2413         }
2414     }
2415
2416     return $res;
2417 }
2418
2419 sub config_list {
2420     my $vmlist = PVE::Cluster::get_vmlist();
2421     my $res = {};
2422     return $res if !$vmlist || !$vmlist->{ids};
2423     my $ids = $vmlist->{ids};
2424     my $nodename = nodename();
2425
2426     foreach my $vmid (keys %$ids) {
2427         my $d = $ids->{$vmid};
2428         next if !$d->{node} || $d->{node} ne $nodename;
2429         next if !$d->{type} || $d->{type} ne 'qemu';
2430         $res->{$vmid}->{exists} = 1;
2431     }
2432     return $res;
2433 }
2434
2435 # test if VM uses local resources (to prevent migration)
2436 sub check_local_resources {
2437     my ($conf, $noerr) = @_;
2438
2439     my @loc_res = ();
2440
2441     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2442     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2443
2444     push @loc_res, "ivshmem" if $conf->{ivshmem};
2445
2446     foreach my $k (keys %$conf) {
2447         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2448         # sockets are safe: they will recreated be on the target side post-migrate
2449         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2450         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2451     }
2452
2453     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2454
2455     return \@loc_res;
2456 }
2457
2458 # check if used storages are available on all nodes (use by migrate)
2459 sub check_storage_availability {
2460     my ($storecfg, $conf, $node) = @_;
2461
2462     PVE::QemuConfig->foreach_volume($conf, sub {
2463         my ($ds, $drive) = @_;
2464
2465         my $volid = $drive->{file};
2466         return if !$volid;
2467
2468         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2469         return if !$sid;
2470
2471         # check if storage is available on both nodes
2472         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2473         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2474
2475         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2476
2477         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2478             if !$scfg->{content}->{$vtype};
2479    });
2480 }
2481
2482 # list nodes where all VM images are available (used by has_feature API)
2483 sub shared_nodes {
2484     my ($conf, $storecfg) = @_;
2485
2486     my $nodelist = PVE::Cluster::get_nodelist();
2487     my $nodehash = { map { $_ => 1 } @$nodelist };
2488     my $nodename = nodename();
2489
2490     PVE::QemuConfig->foreach_volume($conf, sub {
2491         my ($ds, $drive) = @_;
2492
2493         my $volid = $drive->{file};
2494         return if !$volid;
2495
2496         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2497         if ($storeid) {
2498             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2499             if ($scfg->{disable}) {
2500                 $nodehash = {};
2501             } elsif (my $avail = $scfg->{nodes}) {
2502                 foreach my $node (keys %$nodehash) {
2503                     delete $nodehash->{$node} if !$avail->{$node};
2504                 }
2505             } elsif (!$scfg->{shared}) {
2506                 foreach my $node (keys %$nodehash) {
2507                     delete $nodehash->{$node} if $node ne $nodename
2508                 }
2509             }
2510         }
2511     });
2512
2513     return $nodehash
2514 }
2515
2516 sub check_local_storage_availability {
2517     my ($conf, $storecfg) = @_;
2518
2519     my $nodelist = PVE::Cluster::get_nodelist();
2520     my $nodehash = { map { $_ => {} } @$nodelist };
2521
2522     PVE::QemuConfig->foreach_volume($conf, sub {
2523         my ($ds, $drive) = @_;
2524
2525         my $volid = $drive->{file};
2526         return if !$volid;
2527
2528         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2529         if ($storeid) {
2530             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2531
2532             if ($scfg->{disable}) {
2533                 foreach my $node (keys %$nodehash) {
2534                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2535                 }
2536             } elsif (my $avail = $scfg->{nodes}) {
2537                 foreach my $node (keys %$nodehash) {
2538                     if (!$avail->{$node}) {
2539                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2540                     }
2541                 }
2542             }
2543         }
2544     });
2545
2546     foreach my $node (values %$nodehash) {
2547         if (my $unavail = $node->{unavailable_storages}) {
2548             $node->{unavailable_storages} = [ sort keys %$unavail ];
2549         }
2550     }
2551
2552     return $nodehash
2553 }
2554
2555 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2556 sub check_running {
2557     my ($vmid, $nocheck, $node) = @_;
2558
2559     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2560     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2561 }
2562
2563 sub vzlist {
2564
2565     my $vzlist = config_list();
2566
2567     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2568
2569     while (defined(my $de = $fd->read)) {
2570         next if $de !~ m/^(\d+)\.pid$/;
2571         my $vmid = $1;
2572         next if !defined($vzlist->{$vmid});
2573         if (my $pid = check_running($vmid)) {
2574             $vzlist->{$vmid}->{pid} = $pid;
2575         }
2576     }
2577
2578     return $vzlist;
2579 }
2580
2581 our $vmstatus_return_properties = {
2582     vmid => get_standard_option('pve-vmid'),
2583     status => {
2584         description => "Qemu process status.",
2585         type => 'string',
2586         enum => ['stopped', 'running'],
2587     },
2588     maxmem => {
2589         description => "Maximum memory in bytes.",
2590         type => 'integer',
2591         optional => 1,
2592         renderer => 'bytes',
2593     },
2594     maxdisk => {
2595         description => "Root disk size in bytes.",
2596         type => 'integer',
2597         optional => 1,
2598         renderer => 'bytes',
2599     },
2600     name => {
2601         description => "VM name.",
2602         type => 'string',
2603         optional => 1,
2604     },
2605     qmpstatus => {
2606         description => "Qemu QMP agent status.",
2607         type => 'string',
2608         optional => 1,
2609     },
2610     pid => {
2611         description => "PID of running qemu process.",
2612         type => 'integer',
2613         optional => 1,
2614     },
2615     uptime => {
2616         description => "Uptime.",
2617         type => 'integer',
2618         optional => 1,
2619         renderer => 'duration',
2620     },
2621     cpus => {
2622         description => "Maximum usable CPUs.",
2623         type => 'number',
2624         optional => 1,
2625     },
2626     lock => {
2627         description => "The current config lock, if any.",
2628         type => 'string',
2629         optional => 1,
2630     },
2631     tags => {
2632         description  => "The current configured tags, if any",
2633         type => 'string',
2634         optional => 1,
2635     },
2636     'running-machine' => {
2637         description => "The currently running machine type (if running).",
2638         type => 'string',
2639         optional => 1,
2640     },
2641     'running-qemu' => {
2642         description => "The currently running QEMU version (if running).",
2643         type => 'string',
2644         optional => 1,
2645     },
2646 };
2647
2648 my $last_proc_pid_stat;
2649
2650 # get VM status information
2651 # This must be fast and should not block ($full == false)
2652 # We only query KVM using QMP if $full == true (this can be slow)
2653 sub vmstatus {
2654     my ($opt_vmid, $full) = @_;
2655
2656     my $res = {};
2657
2658     my $storecfg = PVE::Storage::config();
2659
2660     my $list = vzlist();
2661     my $defaults = load_defaults();
2662
2663     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2664
2665     my $cpucount = $cpuinfo->{cpus} || 1;
2666
2667     foreach my $vmid (keys %$list) {
2668         next if $opt_vmid && ($vmid ne $opt_vmid);
2669
2670         my $conf = PVE::QemuConfig->load_config($vmid);
2671
2672         my $d = { vmid => int($vmid) };
2673         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2674
2675         # fixme: better status?
2676         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2677
2678         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2679         if (defined($size)) {
2680             $d->{disk} = 0; # no info available
2681             $d->{maxdisk} = $size;
2682         } else {
2683             $d->{disk} = 0;
2684             $d->{maxdisk} = 0;
2685         }
2686
2687         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2688             * ($conf->{cores} || $defaults->{cores});
2689         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2690         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2691
2692         $d->{name} = $conf->{name} || "VM $vmid";
2693         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2694             : $defaults->{memory}*(1024*1024);
2695
2696         if ($conf->{balloon}) {
2697             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2698             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2699                 : $defaults->{shares};
2700         }
2701
2702         $d->{uptime} = 0;
2703         $d->{cpu} = 0;
2704         $d->{mem} = 0;
2705
2706         $d->{netout} = 0;
2707         $d->{netin} = 0;
2708
2709         $d->{diskread} = 0;
2710         $d->{diskwrite} = 0;
2711
2712         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2713
2714         $d->{serial} = 1 if conf_has_serial($conf);
2715         $d->{lock} = $conf->{lock} if $conf->{lock};
2716         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2717
2718         $res->{$vmid} = $d;
2719     }
2720
2721     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2722     foreach my $dev (keys %$netdev) {
2723         next if $dev !~ m/^tap([1-9]\d*)i/;
2724         my $vmid = $1;
2725         my $d = $res->{$vmid};
2726         next if !$d;
2727
2728         $d->{netout} += $netdev->{$dev}->{receive};
2729         $d->{netin} += $netdev->{$dev}->{transmit};
2730
2731         if ($full) {
2732             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2733             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2734         }
2735
2736     }
2737
2738     my $ctime = gettimeofday;
2739
2740     foreach my $vmid (keys %$list) {
2741
2742         my $d = $res->{$vmid};
2743         my $pid = $d->{pid};
2744         next if !$pid;
2745
2746         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2747         next if !$pstat; # not running
2748
2749         my $used = $pstat->{utime} + $pstat->{stime};
2750
2751         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2752
2753         if ($pstat->{vsize}) {
2754             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2755         }
2756
2757         my $old = $last_proc_pid_stat->{$pid};
2758         if (!$old) {
2759             $last_proc_pid_stat->{$pid} = {
2760                 time => $ctime,
2761                 used => $used,
2762                 cpu => 0,
2763             };
2764             next;
2765         }
2766
2767         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2768
2769         if ($dtime > 1000) {
2770             my $dutime = $used -  $old->{used};
2771
2772             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2773             $last_proc_pid_stat->{$pid} = {
2774                 time => $ctime,
2775                 used => $used,
2776                 cpu => $d->{cpu},
2777             };
2778         } else {
2779             $d->{cpu} = $old->{cpu};
2780         }
2781     }
2782
2783     return $res if !$full;
2784
2785     my $qmpclient = PVE::QMPClient->new();
2786
2787     my $ballooncb = sub {
2788         my ($vmid, $resp) = @_;
2789
2790         my $info = $resp->{'return'};
2791         return if !$info->{max_mem};
2792
2793         my $d = $res->{$vmid};
2794
2795         # use memory assigned to VM
2796         $d->{maxmem} = $info->{max_mem};
2797         $d->{balloon} = $info->{actual};
2798
2799         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2800             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2801             $d->{freemem} = $info->{free_mem};
2802         }
2803
2804         $d->{ballooninfo} = $info;
2805     };
2806
2807     my $blockstatscb = sub {
2808         my ($vmid, $resp) = @_;
2809         my $data = $resp->{'return'} || [];
2810         my $totalrdbytes = 0;
2811         my $totalwrbytes = 0;
2812
2813         for my $blockstat (@$data) {
2814             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2815             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2816
2817             $blockstat->{device} =~ s/drive-//;
2818             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2819         }
2820         $res->{$vmid}->{diskread} = $totalrdbytes;
2821         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2822     };
2823
2824     my $machinecb = sub {
2825         my ($vmid, $resp) = @_;
2826         my $data = $resp->{'return'} || [];
2827
2828         $res->{$vmid}->{'running-machine'} =
2829             PVE::QemuServer::Machine::current_from_query_machines($data);
2830     };
2831
2832     my $versioncb = sub {
2833         my ($vmid, $resp) = @_;
2834         my $data = $resp->{'return'} // {};
2835         my $version = 'unknown';
2836
2837         if (my $v = $data->{qemu}) {
2838             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
2839         }
2840
2841         $res->{$vmid}->{'running-qemu'} = $version;
2842     };
2843
2844     my $statuscb = sub {
2845         my ($vmid, $resp) = @_;
2846
2847         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2848         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
2849         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
2850         # this fails if ballon driver is not loaded, so this must be
2851         # the last commnand (following command are aborted if this fails).
2852         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2853
2854         my $status = 'unknown';
2855         if (!defined($status = $resp->{'return'}->{status})) {
2856             warn "unable to get VM status\n";
2857             return;
2858         }
2859
2860         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2861     };
2862
2863     foreach my $vmid (keys %$list) {
2864         next if $opt_vmid && ($vmid ne $opt_vmid);
2865         next if !$res->{$vmid}->{pid}; # not running
2866         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2867     }
2868
2869     $qmpclient->queue_execute(undef, 2);
2870
2871     foreach my $vmid (keys %$list) {
2872         next if $opt_vmid && ($vmid ne $opt_vmid);
2873         next if !$res->{$vmid}->{pid}; #not running
2874
2875         # we can't use the $qmpclient since it might have already aborted on
2876         # 'query-balloon', but this might also fail for older versions...
2877         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
2878         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
2879     }
2880
2881     foreach my $vmid (keys %$list) {
2882         next if $opt_vmid && ($vmid ne $opt_vmid);
2883         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2884     }
2885
2886     return $res;
2887 }
2888
2889 sub conf_has_serial {
2890     my ($conf) = @_;
2891
2892     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2893         if ($conf->{"serial$i"}) {
2894             return 1;
2895         }
2896     }
2897
2898     return 0;
2899 }
2900
2901 sub conf_has_audio {
2902     my ($conf, $id) = @_;
2903
2904     $id //= 0;
2905     my $audio = $conf->{"audio$id"};
2906     return if !defined($audio);
2907
2908     my $audioproperties = parse_property_string($audio_fmt, $audio);
2909     my $audiodriver = $audioproperties->{driver} // 'spice';
2910
2911     return {
2912         dev => $audioproperties->{device},
2913         dev_id => "audiodev$id",
2914         backend => $audiodriver,
2915         backend_id => "$audiodriver-backend${id}",
2916     };
2917 }
2918
2919 sub audio_devs {
2920     my ($audio, $audiopciaddr, $machine_version) = @_;
2921
2922     my $devs = [];
2923
2924     my $id = $audio->{dev_id};
2925     my $audiodev = "";
2926     if (min_version($machine_version, 4, 2)) {
2927         $audiodev = ",audiodev=$audio->{backend_id}";
2928     }
2929
2930     if ($audio->{dev} eq 'AC97') {
2931         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
2932     } elsif ($audio->{dev} =~ /intel\-hda$/) {
2933         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
2934         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
2935         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
2936     } else {
2937         die "unkown audio device '$audio->{dev}', implement me!";
2938     }
2939
2940     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
2941
2942     return $devs;
2943 }
2944
2945 sub vga_conf_has_spice {
2946     my ($vga) = @_;
2947
2948     my $vgaconf = parse_vga($vga);
2949     my $vgatype = $vgaconf->{type};
2950     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
2951
2952     return $1 || 1;
2953 }
2954
2955 sub is_native($) {
2956     my ($arch) = @_;
2957     return get_host_arch() eq $arch;
2958 }
2959
2960 sub get_vm_arch {
2961     my ($conf) = @_;
2962     return $conf->{arch} // get_host_arch();
2963 }
2964
2965 my $default_machines = {
2966     x86_64 => 'pc',
2967     aarch64 => 'virt',
2968 };
2969
2970 sub get_installed_machine_version {
2971     my ($kvmversion) = @_;
2972     $kvmversion = kvm_user_version() if !defined($kvmversion);
2973     $kvmversion =~ m/^(\d+\.\d+)/;
2974     return $1;
2975 }
2976
2977 sub windows_get_pinned_machine_version {
2978     my ($machine, $base_version, $kvmversion) = @_;
2979
2980     my $pin_version = $base_version;
2981     if (!defined($base_version) ||
2982         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
2983     ) {
2984         $pin_version = get_installed_machine_version($kvmversion);
2985     }
2986     if (!$machine || $machine eq 'pc') {
2987         $machine = "pc-i440fx-$pin_version";
2988     } elsif ($machine eq 'q35') {
2989         $machine = "pc-q35-$pin_version";
2990     } elsif ($machine eq 'virt') {
2991         $machine = "virt-$pin_version";
2992     } else {
2993         warn "unknown machine type '$machine', not touching that!\n";
2994     }
2995
2996     return $machine;
2997 }
2998
2999 sub get_vm_machine {
3000     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3001
3002     my $machine = $forcemachine || $conf->{machine};
3003
3004     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3005         $kvmversion //= kvm_user_version();
3006         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3007         # layout which confuses windows quite a bit and may result in various regressions..
3008         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3009         if (windows_version($conf->{ostype})) {
3010             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3011         }
3012         $arch //= 'x86_64';
3013         $machine ||= $default_machines->{$arch};
3014         if ($add_pve_version) {
3015             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3016             $machine .= "+pve$pvever";
3017         }
3018     }
3019
3020     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3021         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3022         $machine = $1 if $is_pxe;
3023
3024         # for version-pinned machines that do not include a pve-version (e.g.
3025         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3026         $machine .= '+pve0';
3027
3028         $machine .= '.pxe' if $is_pxe;
3029     }
3030
3031     return $machine;
3032 }
3033
3034 sub get_ovmf_files($) {
3035     my ($arch) = @_;
3036
3037     my $ovmf = $OVMF->{$arch}
3038         or die "no OVMF images known for architecture '$arch'\n";
3039
3040     return @$ovmf;
3041 }
3042
3043 my $Arch2Qemu = {
3044     aarch64 => '/usr/bin/qemu-system-aarch64',
3045     x86_64 => '/usr/bin/qemu-system-x86_64',
3046 };
3047 sub get_command_for_arch($) {
3048     my ($arch) = @_;
3049     return '/usr/bin/kvm' if is_native($arch);
3050
3051     my $cmd = $Arch2Qemu->{$arch}
3052         or die "don't know how to emulate architecture '$arch'\n";
3053     return $cmd;
3054 }
3055
3056 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3057 # to use in a QEMU command line (-cpu element), first array_intersect the result
3058 # of query_supported_ with query_understood_. This is necessary because:
3059 #
3060 # a) query_understood_ returns flags the host cannot use and
3061 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3062 #    flags, but CPU settings - with most of them being flags. Those settings
3063 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3064 #
3065 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3066 # expensive. If you need the value returned from this, you can get it much
3067 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3068 # $accel being 'kvm' or 'tcg'.
3069 #
3070 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3071 # changes, automatically populating pmxcfs.
3072 #
3073 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3074 # since kvm and tcg machines support different flags
3075 #
3076 sub query_supported_cpu_flags {
3077     my ($arch) = @_;
3078
3079     $arch //= get_host_arch();
3080     my $default_machine = $default_machines->{$arch};
3081
3082     my $flags = {};
3083
3084     # FIXME: Once this is merged, the code below should work for ARM as well:
3085     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3086     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3087         $arch eq "aarch64";
3088
3089     my $kvm_supported = defined(kvm_version());
3090     my $qemu_cmd = get_command_for_arch($arch);
3091     my $fakevmid = -1;
3092     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3093
3094     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3095     my $query_supported_run_qemu = sub {
3096         my ($kvm) = @_;
3097
3098         my $flags = {};
3099         my $cmd = [
3100             $qemu_cmd,
3101             '-machine', $default_machine,
3102             '-display', 'none',
3103             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3104             '-mon', 'chardev=qmp,mode=control',
3105             '-pidfile', $pidfile,
3106             '-S', '-daemonize'
3107         ];
3108
3109         if (!$kvm) {
3110             push @$cmd, '-accel', 'tcg';
3111         }
3112
3113         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3114         die "QEMU flag querying VM exited with code " . $rc if $rc;
3115
3116         eval {
3117             my $cmd_result = mon_cmd(
3118                 $fakevmid,
3119                 'query-cpu-model-expansion',
3120                 type => 'full',
3121                 model => { name => 'host' }
3122             );
3123
3124             my $props = $cmd_result->{model}->{props};
3125             foreach my $prop (keys %$props) {
3126                 next if $props->{$prop} ne '1';
3127                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3128                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3129                 # We only keep those with underscores, to match /proc/cpuinfo
3130                 $prop =~ s/\.|-/_/g;
3131                 $flags->{$prop} = 1;
3132             }
3133         };
3134         my $err = $@;
3135
3136         # force stop with 10 sec timeout and 'nocheck'
3137         # always stop, even if QMP failed
3138         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3139
3140         die $err if $err;
3141
3142         return [ sort keys %$flags ];
3143     };
3144
3145     # We need to query QEMU twice, since KVM and TCG have different supported flags
3146     PVE::QemuConfig->lock_config($fakevmid, sub {
3147         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3148         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3149
3150         if ($kvm_supported) {
3151             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3152             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3153         }
3154     });
3155
3156     return $flags;
3157 }
3158
3159 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3160 my $understood_cpu_flag_dir = "/usr/share/kvm";
3161 sub query_understood_cpu_flags {
3162     my $arch = get_host_arch();
3163     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3164
3165     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3166         if ! -e $filepath;
3167
3168     my $raw = file_get_contents($filepath);
3169     $raw =~ s/^\s+|\s+$//g;
3170     my @flags = split(/\s+/, $raw);
3171
3172     return \@flags;
3173 }
3174
3175 sub config_to_command {
3176     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3177         $pbs_backing) = @_;
3178
3179     my $cmd = [];
3180     my $globalFlags = [];
3181     my $machineFlags = [];
3182     my $rtcFlags = [];
3183     my $devices = [];
3184     my $pciaddr = '';
3185     my $bridges = {};
3186     my $ostype = $conf->{ostype};
3187     my $winversion = windows_version($ostype);
3188     my $kvm = $conf->{kvm};
3189     my $nodename = nodename();
3190
3191     my $arch = get_vm_arch($conf);
3192     my $kvm_binary = get_command_for_arch($arch);
3193     my $kvmver = kvm_user_version($kvm_binary);
3194
3195     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3196         $kvmver //= "undefined";
3197         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3198     }
3199
3200     my $add_pve_version = min_version($kvmver, 4, 1);
3201
3202     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3203     my $machine_version = extract_version($machine_type, $kvmver);
3204     $kvm //= 1 if is_native($arch);
3205
3206     $machine_version =~ m/(\d+)\.(\d+)/;
3207     my ($machine_major, $machine_minor) = ($1, $2);
3208
3209     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3210         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3211     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3212         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3213             ." please upgrade node '$nodename'\n"
3214     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3215         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3216         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3217             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3218             ." node '$nodename'\n";
3219     }
3220
3221     # if a specific +pve version is required for a feature, use $version_guard
3222     # instead of min_version to allow machines to be run with the minimum
3223     # required version
3224     my $required_pve_version = 0;
3225     my $version_guard = sub {
3226         my ($major, $minor, $pve) = @_;
3227         return 0 if !min_version($machine_version, $major, $minor, $pve);
3228         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3229         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3230         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3231         return 1;
3232     };
3233
3234     if ($kvm && !defined kvm_version()) {
3235         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3236             ." or enable in BIOS.\n";
3237     }
3238
3239     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3240     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3241     my $use_old_bios_files = undef;
3242     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3243
3244     my $cpuunits = defined($conf->{cpuunits}) ?
3245             $conf->{cpuunits} : $defaults->{cpuunits};
3246
3247     push @$cmd, $kvm_binary;
3248
3249     push @$cmd, '-id', $vmid;
3250
3251     my $vmname = $conf->{name} || "vm$vmid";
3252
3253     push @$cmd, '-name', $vmname;
3254
3255     push @$cmd, '-no-shutdown';
3256
3257     my $use_virtio = 0;
3258
3259     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3260     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3261     push @$cmd, '-mon', "chardev=qmp,mode=control";
3262
3263     if (min_version($machine_version, 2, 12)) {
3264         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3265         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3266     }
3267
3268     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3269
3270     push @$cmd, '-daemonize';
3271
3272     if ($conf->{smbios1}) {
3273         my $smbios_conf = parse_smbios1($conf->{smbios1});
3274         if ($smbios_conf->{base64}) {
3275             # Do not pass base64 flag to qemu
3276             delete $smbios_conf->{base64};
3277             my $smbios_string = "";
3278             foreach my $key (keys %$smbios_conf) {
3279                 my $value;
3280                 if ($key eq "uuid") {
3281                     $value = $smbios_conf->{uuid}
3282                 } else {
3283                     $value = decode_base64($smbios_conf->{$key});
3284                 }
3285                 # qemu accepts any binary data, only commas need escaping by double comma
3286                 $value =~ s/,/,,/g;
3287                 $smbios_string .= "," . $key . "=" . $value if $value;
3288             }
3289             push @$cmd, '-smbios', "type=1" . $smbios_string;
3290         } else {
3291             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3292         }
3293     }
3294
3295     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3296         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3297         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3298
3299         my ($path, $format);
3300         my $read_only_str = '';
3301         if (my $efidisk = $conf->{efidisk0}) {
3302             my $d = parse_drive('efidisk0', $efidisk);
3303             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3304             $format = $d->{format};
3305             if ($storeid) {
3306                 $path = PVE::Storage::path($storecfg, $d->{file});
3307                 if (!defined($format)) {
3308                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3309                     $format = qemu_img_format($scfg, $volname);
3310                 }
3311             } else {
3312                 $path = $d->{file};
3313                 die "efidisk format must be specified\n"
3314                     if !defined($format);
3315             }
3316
3317             $read_only_str = ',readonly=on' if drive_is_read_only($conf, $d);
3318         } else {
3319             warn "no efidisk configured! Using temporary efivars disk.\n";
3320             $path = "/tmp/$vmid-ovmf.fd";
3321             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3322             $format = 'raw';
3323         }
3324
3325         my $size_str = "";
3326
3327         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3328             $size_str = ",size=" . (-s $ovmf_vars);
3329         }
3330
3331         # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3332         my $cache = "";
3333         if ($path =~ m/^rbd:/) {
3334                 $cache = ',cache=writeback';
3335                 $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3336         }
3337
3338         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code";
3339         push @$cmd, '-drive', "if=pflash,unit=1$cache,format=$format,id=drive-efidisk0$size_str,file=${path}${read_only_str}";
3340     }
3341
3342     # load q35 config
3343     if ($q35) {
3344         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3345         if (min_version($machine_version, 4, 0)) {
3346             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3347         } else {
3348             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3349         }
3350     }
3351
3352     if ($conf->{vmgenid}) {
3353         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3354     }
3355
3356     # add usb controllers
3357     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3358         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3359     push @$devices, @usbcontrollers if @usbcontrollers;
3360     my $vga = parse_vga($conf->{vga});
3361
3362     my $qxlnum = vga_conf_has_spice($conf->{vga});
3363     $vga->{type} = 'qxl' if $qxlnum;
3364
3365     if (!$vga->{type}) {
3366         if ($arch eq 'aarch64') {
3367             $vga->{type} = 'virtio';
3368         } elsif (min_version($machine_version, 2, 9)) {
3369             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3370         } else {
3371             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3372         }
3373     }
3374
3375     # enable absolute mouse coordinates (needed by vnc)
3376     my $tablet;
3377     if (defined($conf->{tablet})) {
3378         $tablet = $conf->{tablet};
3379     } else {
3380         $tablet = $defaults->{tablet};
3381         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3382         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3383     }
3384
3385     if ($tablet) {
3386         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3387         my $kbd = print_keyboarddevice_full($conf, $arch);
3388         push @$devices, '-device', $kbd if defined($kbd);
3389     }
3390
3391     my $bootorder = device_bootorder($conf);
3392
3393     # host pci device passthrough
3394     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3395         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3396
3397     # usb devices
3398     my $usb_dev_features = {};
3399     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3400
3401     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3402         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3403     push @$devices, @usbdevices if @usbdevices;
3404
3405     # serial devices
3406     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3407         if (my $path = $conf->{"serial$i"}) {
3408             if ($path eq 'socket') {
3409                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3410                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3411                 # On aarch64, serial0 is the UART device. Qemu only allows
3412                 # connecting UART devices via the '-serial' command line, as
3413                 # the device has a fixed slot on the hardware...
3414                 if ($arch eq 'aarch64' && $i == 0) {
3415                     push @$devices, '-serial', "chardev:serial$i";
3416                 } else {
3417                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3418                 }
3419             } else {
3420                 die "no such serial device\n" if ! -c $path;
3421                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3422                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3423             }
3424         }
3425     }
3426
3427     # parallel devices
3428     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3429         if (my $path = $conf->{"parallel$i"}) {
3430             die "no such parallel device\n" if ! -c $path;
3431             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3432             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3433             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3434         }
3435     }
3436
3437     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3438         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3439         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3440         push @$devices, @$audio_devs;
3441     }
3442
3443     my $sockets = 1;
3444     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3445     $sockets = $conf->{sockets} if  $conf->{sockets};
3446
3447     my $cores = $conf->{cores} || 1;
3448
3449     my $maxcpus = $sockets * $cores;
3450
3451     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3452
3453     my $allowed_vcpus = $cpuinfo->{cpus};
3454
3455     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3456         if ($allowed_vcpus < $maxcpus);
3457
3458     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3459
3460         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3461         for (my $i = 2; $i <= $vcpus; $i++)  {
3462             my $cpustr = print_cpu_device($conf,$i);
3463             push @$cmd, '-device', $cpustr;
3464         }
3465
3466     } else {
3467
3468         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3469     }
3470     push @$cmd, '-nodefaults';
3471
3472     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3473
3474     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3475
3476     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3477
3478     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3479         push @$devices, '-device', print_vga_device(
3480             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3481         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3482         push @$cmd,  '-vnc', "unix:$socket,password=on";
3483     } else {
3484         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3485         push @$cmd, '-nographic';
3486     }
3487
3488     # time drift fix
3489     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3490     my $useLocaltime = $conf->{localtime};
3491
3492     if ($winversion >= 5) { # windows
3493         $useLocaltime = 1 if !defined($conf->{localtime});
3494
3495         # use time drift fix when acpi is enabled
3496         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3497             $tdf = 1 if !defined($conf->{tdf});
3498         }
3499     }
3500
3501     if ($winversion >= 6) {
3502         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3503         push @$cmd, '-no-hpet';
3504     }
3505
3506     push @$rtcFlags, 'driftfix=slew' if $tdf;
3507
3508     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3509         push @$rtcFlags, "base=$conf->{startdate}";
3510     } elsif ($useLocaltime) {
3511         push @$rtcFlags, 'base=localtime';
3512     }
3513
3514     if ($forcecpu) {
3515         push @$cmd, '-cpu', $forcecpu;
3516     } else {
3517         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3518     }
3519
3520     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3521
3522     push @$cmd, '-S' if $conf->{freeze};
3523
3524     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3525
3526     my $guest_agent = parse_guest_agent($conf);
3527
3528     if ($guest_agent->{enabled}) {
3529         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3530         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3531
3532         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3533             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3534             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3535             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3536         } elsif ($guest_agent->{type} eq 'isa') {
3537             push @$devices, '-device', "isa-serial,chardev=qga0";
3538         }
3539     }
3540
3541     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3542     if ($rng && $version_guard->(4, 1, 2)) {
3543         check_rng_source($rng->{source});
3544
3545         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3546         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3547         my $limiter_str = "";
3548         if ($max_bytes) {
3549             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3550         }
3551
3552         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3553         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3554         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3555     }
3556
3557     my $spice_port;
3558
3559     if ($qxlnum) {
3560         if ($qxlnum > 1) {
3561             if ($winversion){
3562                 for (my $i = 1; $i < $qxlnum; $i++){
3563                     push @$devices, '-device', print_vga_device(
3564                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3565                 }
3566             } else {
3567                 # assume other OS works like Linux
3568                 my ($ram, $vram) = ("134217728", "67108864");
3569                 if ($vga->{memory}) {
3570                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3571                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3572                 }
3573                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3574                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3575             }
3576         }
3577
3578         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3579
3580         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3581         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3582         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3583
3584         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3585         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3586         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3587
3588         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3589         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3590
3591         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3592         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3593         if ($spice_enhancement->{foldersharing}) {
3594             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3595             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3596         }
3597
3598         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3599         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3600             if $spice_enhancement->{videostreaming};
3601
3602         push @$devices, '-spice', "$spice_opts";
3603     }
3604
3605     # enable balloon by default, unless explicitly disabled
3606     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3607         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3608         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3609     }
3610
3611     if ($conf->{watchdog}) {
3612         my $wdopts = parse_watchdog($conf->{watchdog});
3613         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3614         my $watchdog = $wdopts->{model} || 'i6300esb';
3615         push @$devices, '-device', "$watchdog$pciaddr";
3616         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3617     }
3618
3619     my $vollist = [];
3620     my $scsicontroller = {};
3621     my $ahcicontroller = {};
3622     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3623
3624     # Add iscsi initiator name if available
3625     if (my $initiator = get_initiator_name()) {
3626         push @$devices, '-iscsi', "initiator-name=$initiator";
3627     }
3628
3629     PVE::QemuConfig->foreach_volume($conf, sub {
3630         my ($ds, $drive) = @_;
3631
3632         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3633             check_volume_storage_type($storecfg, $drive->{file});
3634             push @$vollist, $drive->{file};
3635         }
3636
3637         # ignore efidisk here, already added in bios/fw handling code above
3638         return if $drive->{interface} eq 'efidisk';
3639
3640         $use_virtio = 1 if $ds =~ m/^virtio/;
3641
3642         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3643
3644         if ($drive->{interface} eq 'virtio'){
3645            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3646         }
3647
3648         if ($drive->{interface} eq 'scsi') {
3649
3650             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3651
3652             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3653                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3654
3655             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3656             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3657
3658             my $iothread = '';
3659             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3660                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3661                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3662             } elsif ($drive->{iothread}) {
3663                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3664             }
3665
3666             my $queues = '';
3667             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3668                 $queues = ",num_queues=$drive->{queues}";
3669             }
3670
3671             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3672                 if !$scsicontroller->{$controller};
3673             $scsicontroller->{$controller}=1;
3674         }
3675
3676         if ($drive->{interface} eq 'sata') {
3677             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3678             $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3679             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3680                 if !$ahcicontroller->{$controller};
3681             $ahcicontroller->{$controller}=1;
3682         }
3683
3684         my $pbs_conf = $pbs_backing->{$ds};
3685         my $pbs_name = undef;
3686         if ($pbs_conf) {
3687             $pbs_name = "drive-$ds-pbs";
3688             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
3689         }
3690
3691         my $drive_cmd = print_drive_commandline_full(
3692             $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
3693
3694         # extra protection for templates, but SATA and IDE don't support it..
3695         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
3696
3697         push @$devices, '-drive',$drive_cmd;
3698         push @$devices, '-device', print_drivedevice_full(
3699             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3700     });
3701
3702     for (my $i = 0; $i < $MAX_NETS; $i++) {
3703         my $netname = "net$i";
3704
3705         next if !$conf->{$netname};
3706         my $d = parse_net($conf->{$netname});
3707         next if !$d;
3708
3709         $use_virtio = 1 if $d->{model} eq 'virtio';
3710
3711         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
3712
3713         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
3714         push @$devices, '-netdev', $netdevfull;
3715
3716         my $netdevicefull = print_netdevice_full(
3717             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
3718
3719         push @$devices, '-device', $netdevicefull;
3720     }
3721
3722     if ($conf->{ivshmem}) {
3723         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3724
3725         my $bus;
3726         if ($q35) {
3727             $bus = print_pcie_addr("ivshmem");
3728         } else {
3729             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3730         }
3731
3732         my $ivshmem_name = $ivshmem->{name} // $vmid;
3733         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3734
3735         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3736         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
3737             .",size=$ivshmem->{size}M";
3738     }
3739
3740     # pci.4 is nested in pci.1
3741     $bridges->{1} = 1 if $bridges->{4};
3742
3743     if (!$q35) {
3744         # add pci bridges
3745         if (min_version($machine_version, 2, 3)) {
3746            $bridges->{1} = 1;
3747            $bridges->{2} = 1;
3748         }
3749
3750         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3751
3752     }
3753
3754     for my $k (sort {$b cmp $a} keys %$bridges) {
3755         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3756
3757         my $k_name = $k;
3758         if ($k == 2 && $legacy_igd) {
3759             $k_name = "$k-igd";
3760         }
3761         $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
3762
3763         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3764         if ($q35) {
3765             # add after -readconfig pve-q35.cfg
3766             splice @$devices, 2, 0, '-device', $devstr;
3767         } else {
3768             unshift @$devices, '-device', $devstr if $k > 0;
3769         }
3770     }
3771
3772     if (!$kvm) {
3773         push @$machineFlags, 'accel=tcg';
3774     }
3775
3776     my $machine_type_min = $machine_type;
3777     if ($add_pve_version) {
3778         $machine_type_min =~ s/\+pve\d+$//;
3779         $machine_type_min .= "+pve$required_pve_version";
3780     }
3781     push @$machineFlags, "type=${machine_type_min}";
3782
3783     push @$cmd, @$devices;
3784     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
3785     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
3786     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
3787
3788     if (my $vmstate = $conf->{vmstate}) {
3789         my $statepath = PVE::Storage::path($storecfg, $vmstate);
3790         push @$vollist, $vmstate;
3791         push @$cmd, '-loadstate', $statepath;
3792         print "activating and using '$vmstate' as vmstate\n";
3793     }
3794
3795     if (PVE::QemuConfig->is_template($conf)) {
3796         # needed to workaround base volumes being read-only
3797         push @$cmd, '-snapshot';
3798     }
3799
3800     # add custom args
3801     if ($conf->{args}) {
3802         my $aa = PVE::Tools::split_args($conf->{args});
3803         push @$cmd, @$aa;
3804     }
3805
3806     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3807 }
3808
3809 sub check_rng_source {
3810     my ($source) = @_;
3811
3812     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3813     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
3814         if ! -e $source;
3815
3816     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
3817     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
3818         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
3819         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
3820         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
3821             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
3822             ." to the host.\n";
3823     }
3824 }
3825
3826 sub spice_port {
3827     my ($vmid) = @_;
3828
3829     my $res = mon_cmd($vmid, 'query-spice');
3830
3831     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3832 }
3833
3834 sub vm_devices_list {
3835     my ($vmid) = @_;
3836
3837     my $res = mon_cmd($vmid, 'query-pci');
3838     my $devices_to_check = [];
3839     my $devices = {};
3840     foreach my $pcibus (@$res) {
3841         push @$devices_to_check, @{$pcibus->{devices}},
3842     }
3843
3844     while (@$devices_to_check) {
3845         my $to_check = [];
3846         for my $d (@$devices_to_check) {
3847             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3848             next if !$d->{'pci_bridge'};
3849
3850             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3851             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3852         }
3853         $devices_to_check = $to_check;
3854     }
3855
3856     my $resblock = mon_cmd($vmid, 'query-block');
3857     foreach my $block (@$resblock) {
3858         if($block->{device} =~ m/^drive-(\S+)/){
3859                 $devices->{$1} = 1;
3860         }
3861     }
3862
3863     my $resmice = mon_cmd($vmid, 'query-mice');
3864     foreach my $mice (@$resmice) {
3865         if ($mice->{name} eq 'QEMU HID Tablet') {
3866             $devices->{tablet} = 1;
3867             last;
3868         }
3869     }
3870
3871     # for usb devices there is no query-usb
3872     # but we can iterate over the entries in
3873     # qom-list path=/machine/peripheral
3874     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3875     foreach my $per (@$resperipheral) {
3876         if ($per->{name} =~ m/^usb\d+$/) {
3877             $devices->{$per->{name}} = 1;
3878         }
3879     }
3880
3881     return $devices;
3882 }
3883
3884 sub vm_deviceplug {
3885     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3886
3887     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3888
3889     my $devices_list = vm_devices_list($vmid);
3890     return 1 if defined($devices_list->{$deviceid});
3891
3892     # add PCI bridge if we need it for the device
3893     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
3894
3895     if ($deviceid eq 'tablet') {
3896
3897         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
3898
3899     } elsif ($deviceid eq 'keyboard') {
3900
3901         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
3902
3903     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3904
3905         die "usb hotplug currently not reliable\n";
3906         # since we can't reliably hot unplug all added usb devices and usb
3907         # passthrough breaks live migration we disable usb hotplugging for now
3908         #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3909
3910     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3911
3912         qemu_iothread_add($vmid, $deviceid, $device);
3913
3914         qemu_driveadd($storecfg, $vmid, $device);
3915         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3916
3917         qemu_deviceadd($vmid, $devicefull);
3918         eval { qemu_deviceaddverify($vmid, $deviceid); };
3919         if (my $err = $@) {
3920             eval { qemu_drivedel($vmid, $deviceid); };
3921             warn $@ if $@;
3922             die $err;
3923         }
3924
3925     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3926
3927
3928         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3929         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3930         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3931
3932         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3933
3934         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3935             qemu_iothread_add($vmid, $deviceid, $device);
3936             $devicefull .= ",iothread=iothread-$deviceid";
3937         }
3938
3939         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3940             $devicefull .= ",num_queues=$device->{queues}";
3941         }
3942
3943         qemu_deviceadd($vmid, $devicefull);
3944         qemu_deviceaddverify($vmid, $deviceid);
3945
3946     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3947
3948         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
3949         qemu_driveadd($storecfg, $vmid, $device);
3950
3951         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3952         eval { qemu_deviceadd($vmid, $devicefull); };
3953         if (my $err = $@) {
3954             eval { qemu_drivedel($vmid, $deviceid); };
3955             warn $@ if $@;
3956             die $err;
3957         }
3958
3959     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3960
3961         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
3962
3963         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
3964         my $use_old_bios_files = undef;
3965         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3966
3967         my $netdevicefull = print_netdevice_full(
3968             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
3969         qemu_deviceadd($vmid, $netdevicefull);
3970         eval {
3971             qemu_deviceaddverify($vmid, $deviceid);
3972             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
3973         };
3974         if (my $err = $@) {
3975             eval { qemu_netdevdel($vmid, $deviceid); };
3976             warn $@ if $@;
3977             die $err;
3978         }
3979
3980     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3981
3982         my $bridgeid = $2;
3983         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3984         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3985
3986         qemu_deviceadd($vmid, $devicefull);
3987         qemu_deviceaddverify($vmid, $deviceid);
3988
3989     } else {
3990         die "can't hotplug device '$deviceid'\n";
3991     }
3992
3993     return 1;
3994 }
3995
3996 # fixme: this should raise exceptions on error!
3997 sub vm_deviceunplug {
3998     my ($vmid, $conf, $deviceid) = @_;
3999
4000     my $devices_list = vm_devices_list($vmid);
4001     return 1 if !defined($devices_list->{$deviceid});
4002
4003     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
4004     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4005
4006     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4007
4008         qemu_devicedel($vmid, $deviceid);
4009
4010     } elsif ($deviceid =~ m/^usb\d+$/) {
4011
4012         die "usb hotplug currently not reliable\n";
4013         # when unplugging usb devices this way, there may be remaining usb
4014         # controllers/hubs so we disable it for now
4015         #qemu_devicedel($vmid, $deviceid);
4016         #qemu_devicedelverify($vmid, $deviceid);
4017
4018     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4019
4020         qemu_devicedel($vmid, $deviceid);
4021         qemu_devicedelverify($vmid, $deviceid);
4022         qemu_drivedel($vmid, $deviceid);
4023         qemu_iothread_del($conf, $vmid, $deviceid);
4024
4025     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4026
4027         qemu_devicedel($vmid, $deviceid);
4028         qemu_devicedelverify($vmid, $deviceid);
4029         qemu_iothread_del($conf, $vmid, $deviceid);
4030
4031     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4032
4033         qemu_devicedel($vmid, $deviceid);
4034         qemu_drivedel($vmid, $deviceid);
4035         qemu_deletescsihw($conf, $vmid, $deviceid);
4036
4037     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4038
4039         qemu_devicedel($vmid, $deviceid);
4040         qemu_devicedelverify($vmid, $deviceid);
4041         qemu_netdevdel($vmid, $deviceid);
4042
4043     } else {
4044         die "can't unplug device '$deviceid'\n";
4045     }
4046
4047     return 1;
4048 }
4049
4050 sub qemu_deviceadd {
4051     my ($vmid, $devicefull) = @_;
4052
4053     $devicefull = "driver=".$devicefull;
4054     my %options =  split(/[=,]/, $devicefull);
4055
4056     mon_cmd($vmid, "device_add" , %options);
4057 }
4058
4059 sub qemu_devicedel {
4060     my ($vmid, $deviceid) = @_;
4061
4062     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4063 }
4064
4065 sub qemu_iothread_add {
4066     my($vmid, $deviceid, $device) = @_;
4067
4068     if ($device->{iothread}) {
4069         my $iothreads = vm_iothreads_list($vmid);
4070         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4071     }
4072 }
4073
4074 sub qemu_iothread_del {
4075     my($conf, $vmid, $deviceid) = @_;
4076
4077     my $confid = $deviceid;
4078     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4079         $confid = 'scsi' . $1;
4080     }
4081     my $device = parse_drive($confid, $conf->{$confid});
4082     if ($device->{iothread}) {
4083         my $iothreads = vm_iothreads_list($vmid);
4084         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4085     }
4086 }
4087
4088 sub qemu_objectadd {
4089     my($vmid, $objectid, $qomtype) = @_;
4090
4091     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4092
4093     return 1;
4094 }
4095
4096 sub qemu_objectdel {
4097     my($vmid, $objectid) = @_;
4098
4099     mon_cmd($vmid, "object-del", id => $objectid);
4100
4101     return 1;
4102 }
4103
4104 sub qemu_driveadd {
4105     my ($storecfg, $vmid, $device) = @_;
4106
4107     my $kvmver = get_running_qemu_version($vmid);
4108     my $io_uring = min_version($kvmver, 6, 0);
4109     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4110     $drive =~ s/\\/\\\\/g;
4111     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4112
4113     # If the command succeeds qemu prints: "OK"
4114     return 1 if $ret =~ m/OK/s;
4115
4116     die "adding drive failed: $ret\n";
4117 }
4118
4119 sub qemu_drivedel {
4120     my($vmid, $deviceid) = @_;
4121
4122     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4123     $ret =~ s/^\s+//;
4124
4125     return 1 if $ret eq "";
4126
4127     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4128     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4129
4130     die "deleting drive $deviceid failed : $ret\n";
4131 }
4132
4133 sub qemu_deviceaddverify {
4134     my ($vmid, $deviceid) = @_;
4135
4136     for (my $i = 0; $i <= 5; $i++) {
4137          my $devices_list = vm_devices_list($vmid);
4138          return 1 if defined($devices_list->{$deviceid});
4139          sleep 1;
4140     }
4141
4142     die "error on hotplug device '$deviceid'\n";
4143 }
4144
4145
4146 sub qemu_devicedelverify {
4147     my ($vmid, $deviceid) = @_;
4148
4149     # need to verify that the device is correctly removed as device_del
4150     # is async and empty return is not reliable
4151
4152     for (my $i = 0; $i <= 5; $i++) {
4153          my $devices_list = vm_devices_list($vmid);
4154          return 1 if !defined($devices_list->{$deviceid});
4155          sleep 1;
4156     }
4157
4158     die "error on hot-unplugging device '$deviceid'\n";
4159 }
4160
4161 sub qemu_findorcreatescsihw {
4162     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4163
4164     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4165
4166     my $scsihwid="$controller_prefix$controller";
4167     my $devices_list = vm_devices_list($vmid);
4168
4169     if(!defined($devices_list->{$scsihwid})) {
4170         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4171     }
4172
4173     return 1;
4174 }
4175
4176 sub qemu_deletescsihw {
4177     my ($conf, $vmid, $opt) = @_;
4178
4179     my $device = parse_drive($opt, $conf->{$opt});
4180
4181     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4182         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4183         return 1;
4184     }
4185
4186     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4187
4188     my $devices_list = vm_devices_list($vmid);
4189     foreach my $opt (keys %{$devices_list}) {
4190         if (is_valid_drivename($opt)) {
4191             my $drive = parse_drive($opt, $conf->{$opt});
4192             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4193                 return 1;
4194             }
4195         }
4196     }
4197
4198     my $scsihwid="scsihw$controller";
4199
4200     vm_deviceunplug($vmid, $conf, $scsihwid);
4201
4202     return 1;
4203 }
4204
4205 sub qemu_add_pci_bridge {
4206     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4207
4208     my $bridges = {};
4209
4210     my $bridgeid;
4211
4212     print_pci_addr($device, $bridges, $arch, $machine_type);
4213
4214     while (my ($k, $v) = each %$bridges) {
4215         $bridgeid = $k;
4216     }
4217     return 1 if !defined($bridgeid) || $bridgeid < 1;
4218
4219     my $bridge = "pci.$bridgeid";
4220     my $devices_list = vm_devices_list($vmid);
4221
4222     if (!defined($devices_list->{$bridge})) {
4223         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4224     }
4225
4226     return 1;
4227 }
4228
4229 sub qemu_set_link_status {
4230     my ($vmid, $device, $up) = @_;
4231
4232     mon_cmd($vmid, "set_link", name => $device,
4233                up => $up ? JSON::true : JSON::false);
4234 }
4235
4236 sub qemu_netdevadd {
4237     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4238
4239     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4240     my %options =  split(/[=,]/, $netdev);
4241
4242     if (defined(my $vhost = $options{vhost})) {
4243         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4244     }
4245
4246     if (defined(my $queues = $options{queues})) {
4247         $options{queues} = $queues + 0;
4248     }
4249
4250     mon_cmd($vmid, "netdev_add",  %options);
4251     return 1;
4252 }
4253
4254 sub qemu_netdevdel {
4255     my ($vmid, $deviceid) = @_;
4256
4257     mon_cmd($vmid, "netdev_del", id => $deviceid);
4258 }
4259
4260 sub qemu_usb_hotplug {
4261     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4262
4263     return if !$device;
4264
4265     # remove the old one first
4266     vm_deviceunplug($vmid, $conf, $deviceid);
4267
4268     # check if xhci controller is necessary and available
4269     if ($device->{usb3}) {
4270
4271         my $devicelist = vm_devices_list($vmid);
4272
4273         if (!$devicelist->{xhci}) {
4274             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4275             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4276         }
4277     }
4278     my $d = parse_usb_device($device->{host});
4279     $d->{usb3} = $device->{usb3};
4280
4281     # add the new one
4282     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4283 }
4284
4285 sub qemu_cpu_hotplug {
4286     my ($vmid, $conf, $vcpus) = @_;
4287
4288     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4289
4290     my $sockets = 1;
4291     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4292     $sockets = $conf->{sockets} if  $conf->{sockets};
4293     my $cores = $conf->{cores} || 1;
4294     my $maxcpus = $sockets * $cores;
4295
4296     $vcpus = $maxcpus if !$vcpus;
4297
4298     die "you can't add more vcpus than maxcpus\n"
4299         if $vcpus > $maxcpus;
4300
4301     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4302
4303     if ($vcpus < $currentvcpus) {
4304
4305         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4306
4307             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4308                 qemu_devicedel($vmid, "cpu$i");
4309                 my $retry = 0;
4310                 my $currentrunningvcpus = undef;
4311                 while (1) {
4312                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4313                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4314                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4315                     $retry++;
4316                     sleep 1;
4317                 }
4318                 #update conf after each succesfull cpu unplug
4319                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4320                 PVE::QemuConfig->write_config($vmid, $conf);
4321             }
4322         } else {
4323             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4324         }
4325
4326         return;
4327     }
4328
4329     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4330     die "vcpus in running vm does not match its configuration\n"
4331         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4332
4333     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4334
4335         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4336             my $cpustr = print_cpu_device($conf, $i);
4337             qemu_deviceadd($vmid, $cpustr);
4338
4339             my $retry = 0;
4340             my $currentrunningvcpus = undef;
4341             while (1) {
4342                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4343                 last if scalar(@{$currentrunningvcpus}) == $i;
4344                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4345                 sleep 1;
4346                 $retry++;
4347             }
4348             #update conf after each succesfull cpu hotplug
4349             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4350             PVE::QemuConfig->write_config($vmid, $conf);
4351         }
4352     } else {
4353
4354         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4355             mon_cmd($vmid, "cpu-add", id => int($i));
4356         }
4357     }
4358 }
4359
4360 sub qemu_block_set_io_throttle {
4361     my ($vmid, $deviceid,
4362         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4363         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4364         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4365         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4366
4367     return if !check_running($vmid) ;
4368
4369     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4370         bps => int($bps),
4371         bps_rd => int($bps_rd),
4372         bps_wr => int($bps_wr),
4373         iops => int($iops),
4374         iops_rd => int($iops_rd),
4375         iops_wr => int($iops_wr),
4376         bps_max => int($bps_max),
4377         bps_rd_max => int($bps_rd_max),
4378         bps_wr_max => int($bps_wr_max),
4379         iops_max => int($iops_max),
4380         iops_rd_max => int($iops_rd_max),
4381         iops_wr_max => int($iops_wr_max),
4382         bps_max_length => int($bps_max_length),
4383         bps_rd_max_length => int($bps_rd_max_length),
4384         bps_wr_max_length => int($bps_wr_max_length),
4385         iops_max_length => int($iops_max_length),
4386         iops_rd_max_length => int($iops_rd_max_length),
4387         iops_wr_max_length => int($iops_wr_max_length),
4388     );
4389
4390 }
4391
4392 sub qemu_block_resize {
4393     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4394
4395     my $running = check_running($vmid);
4396
4397     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4398
4399     return if !$running;
4400
4401     my $padding = (1024 - $size % 1024) % 1024;
4402     $size = $size + $padding;
4403
4404     mon_cmd(
4405         $vmid,
4406         "block_resize",
4407         device => $deviceid,
4408         size => int($size),
4409         timeout => 60,
4410     );
4411 }
4412
4413 sub qemu_volume_snapshot {
4414     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4415
4416     my $running = check_running($vmid);
4417
4418     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4419         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4420     } else {
4421         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4422     }
4423 }
4424
4425 sub qemu_volume_snapshot_delete {
4426     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4427
4428     my $running = check_running($vmid);
4429
4430     if($running) {
4431
4432         $running = undef;
4433         my $conf = PVE::QemuConfig->load_config($vmid);
4434         PVE::QemuConfig->foreach_volume($conf, sub {
4435             my ($ds, $drive) = @_;
4436             $running = 1 if $drive->{file} eq $volid;
4437         });
4438     }
4439
4440     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4441         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4442     } else {
4443         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4444     }
4445 }
4446
4447 sub set_migration_caps {
4448     my ($vmid, $savevm) = @_;
4449
4450     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4451
4452     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4453     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4454
4455     my $cap_ref = [];
4456
4457     my $enabled_cap = {
4458         "auto-converge" => 1,
4459         "xbzrle" => 1,
4460         "x-rdma-pin-all" => 0,
4461         "zero-blocks" => 0,
4462         "compress" => 0,
4463         "dirty-bitmaps" => $dirty_bitmaps,
4464     };
4465
4466     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4467
4468     for my $supported_capability (@$supported_capabilities) {
4469         push @$cap_ref, {
4470             capability => $supported_capability->{capability},
4471             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4472         };
4473     }
4474
4475     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4476 }
4477
4478 sub foreach_volid {
4479     my ($conf, $func, @param) = @_;
4480
4481     my $volhash = {};
4482
4483     my $test_volid = sub {
4484         my ($key, $drive, $snapname) = @_;
4485
4486         my $volid = $drive->{file};
4487         return if !$volid;
4488
4489         $volhash->{$volid}->{cdrom} //= 1;
4490         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4491
4492         my $replicate = $drive->{replicate} // 1;
4493         $volhash->{$volid}->{replicate} //= 0;
4494         $volhash->{$volid}->{replicate} = 1 if $replicate;
4495
4496         $volhash->{$volid}->{shared} //= 0;
4497         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4498
4499         $volhash->{$volid}->{referenced_in_config} //= 0;
4500         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4501
4502         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4503             if defined($snapname);
4504
4505         my $size = $drive->{size};
4506         $volhash->{$volid}->{size} //= $size if $size;
4507
4508         $volhash->{$volid}->{is_vmstate} //= 0;
4509         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4510
4511         $volhash->{$volid}->{is_unused} //= 0;
4512         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4513
4514         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4515     };
4516
4517     my $include_opts = {
4518         extra_keys => ['vmstate'],
4519         include_unused => 1,
4520     };
4521
4522     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4523     foreach my $snapname (keys %{$conf->{snapshots}}) {
4524         my $snap = $conf->{snapshots}->{$snapname};
4525         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4526     }
4527
4528     foreach my $volid (keys %$volhash) {
4529         &$func($volid, $volhash->{$volid}, @param);
4530     }
4531 }
4532
4533 my $fast_plug_option = {
4534     'lock' => 1,
4535     'name' => 1,
4536     'onboot' => 1,
4537     'shares' => 1,
4538     'startup' => 1,
4539     'description' => 1,
4540     'protection' => 1,
4541     'vmstatestorage' => 1,
4542     'hookscript' => 1,
4543     'tags' => 1,
4544 };
4545
4546 # hotplug changes in [PENDING]
4547 # $selection hash can be used to only apply specified options, for
4548 # example: { cores => 1 } (only apply changed 'cores')
4549 # $errors ref is used to return error messages
4550 sub vmconfig_hotplug_pending {
4551     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4552
4553     my $defaults = load_defaults();
4554     my $arch = get_vm_arch($conf);
4555     my $machine_type = get_vm_machine($conf, undef, $arch);
4556
4557     # commit values which do not have any impact on running VM first
4558     # Note: those option cannot raise errors, we we do not care about
4559     # $selection and always apply them.
4560
4561     my $add_error = sub {
4562         my ($opt, $msg) = @_;
4563         $errors->{$opt} = "hotplug problem - $msg";
4564     };
4565
4566     my $changes = 0;
4567     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4568         if ($fast_plug_option->{$opt}) {
4569             $conf->{$opt} = $conf->{pending}->{$opt};
4570             delete $conf->{pending}->{$opt};
4571             $changes = 1;
4572         }
4573     }
4574
4575     if ($changes) {
4576         PVE::QemuConfig->write_config($vmid, $conf);
4577     }
4578
4579     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4580
4581     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4582     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4583     foreach my $opt (sort keys %$pending_delete_hash) {
4584         next if $selection && !$selection->{$opt};
4585         my $force = $pending_delete_hash->{$opt}->{force};
4586         eval {
4587             if ($opt eq 'hotplug') {
4588                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4589             } elsif ($opt eq 'tablet') {
4590                 die "skip\n" if !$hotplug_features->{usb};
4591                 if ($defaults->{tablet}) {
4592                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4593                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4594                         if $arch eq 'aarch64';
4595                 } else {
4596                     vm_deviceunplug($vmid, $conf, 'tablet');
4597                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4598                 }
4599             } elsif ($opt =~ m/^usb\d+/) {
4600                 die "skip\n";
4601                 # since we cannot reliably hot unplug usb devices we are disabling it
4602                 #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4603                 #vm_deviceunplug($vmid, $conf, $opt);
4604             } elsif ($opt eq 'vcpus') {
4605                 die "skip\n" if !$hotplug_features->{cpu};
4606                 qemu_cpu_hotplug($vmid, $conf, undef);
4607             } elsif ($opt eq 'balloon') {
4608                 # enable balloon device is not hotpluggable
4609                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4610                 # here we reset the ballooning value to memory
4611                 my $balloon = $conf->{memory} || $defaults->{memory};
4612                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4613             } elsif ($fast_plug_option->{$opt}) {
4614                 # do nothing
4615             } elsif ($opt =~ m/^net(\d+)$/) {
4616                 die "skip\n" if !$hotplug_features->{network};
4617                 vm_deviceunplug($vmid, $conf, $opt);
4618             } elsif (is_valid_drivename($opt)) {
4619                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4620                 vm_deviceunplug($vmid, $conf, $opt);
4621                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4622             } elsif ($opt =~ m/^memory$/) {
4623                 die "skip\n" if !$hotplug_features->{memory};
4624                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4625             } elsif ($opt eq 'cpuunits') {
4626                 $cgroup->change_cpu_shares(undef, $defaults->{cpuunits});
4627             } elsif ($opt eq 'cpulimit') {
4628                 $cgroup->change_cpu_quota(-1, 100000);
4629             } else {
4630                 die "skip\n";
4631             }
4632         };
4633         if (my $err = $@) {
4634             &$add_error($opt, $err) if $err ne "skip\n";
4635         } else {
4636             delete $conf->{$opt};
4637             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4638         }
4639     }
4640
4641     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4642     $apply_pending_cloudinit = sub {
4643         return if $apply_pending_cloudinit_done; # once is enough
4644         $apply_pending_cloudinit_done = 1; # once is enough
4645
4646         my ($key, $value) = @_;
4647
4648         my @cloudinit_opts = keys %$confdesc_cloudinit;
4649         foreach my $opt (keys %{$conf->{pending}}) {
4650             next if !grep { $_ eq $opt } @cloudinit_opts;
4651             $conf->{$opt} = delete $conf->{pending}->{$opt};
4652         }
4653
4654         my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4655         foreach my $opt (sort keys %$pending_delete_hash) {
4656             next if !grep { $_ eq $opt } @cloudinit_opts;
4657             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4658             delete $conf->{$opt};
4659         }
4660
4661         my $new_conf = { %$conf };
4662         $new_conf->{$key} = $value;
4663         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4664     };
4665
4666     foreach my $opt (keys %{$conf->{pending}}) {
4667         next if $selection && !$selection->{$opt};
4668         my $value = $conf->{pending}->{$opt};
4669         eval {
4670             if ($opt eq 'hotplug') {
4671                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4672             } elsif ($opt eq 'tablet') {
4673                 die "skip\n" if !$hotplug_features->{usb};
4674                 if ($value == 1) {
4675                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4676                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4677                         if $arch eq 'aarch64';
4678                 } elsif ($value == 0) {
4679                     vm_deviceunplug($vmid, $conf, 'tablet');
4680                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4681                 }
4682             } elsif ($opt =~ m/^usb\d+$/) {
4683                 die "skip\n";
4684                 # since we cannot reliably hot unplug usb devices we disable it for now
4685                 #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4686                 #my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4687                 #die "skip\n" if !$d;
4688                 #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4689             } elsif ($opt eq 'vcpus') {
4690                 die "skip\n" if !$hotplug_features->{cpu};
4691                 qemu_cpu_hotplug($vmid, $conf, $value);
4692             } elsif ($opt eq 'balloon') {
4693                 # enable/disable balloning device is not hotpluggable
4694                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4695                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4696                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4697
4698                 # allow manual ballooning if shares is set to zero
4699                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4700                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4701                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4702                 }
4703             } elsif ($opt =~ m/^net(\d+)$/) {
4704                 # some changes can be done without hotplug
4705                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4706                                     $vmid, $opt, $value, $arch, $machine_type);
4707             } elsif (is_valid_drivename($opt)) {
4708                 die "skip\n" if $opt eq 'efidisk0';
4709                 # some changes can be done without hotplug
4710                 my $drive = parse_drive($opt, $value);
4711                 if (drive_is_cloudinit($drive)) {
4712                     &$apply_pending_cloudinit($opt, $value);
4713                 }
4714                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4715                                      $vmid, $opt, $value, $arch, $machine_type);
4716             } elsif ($opt =~ m/^memory$/) { #dimms
4717                 die "skip\n" if !$hotplug_features->{memory};
4718                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4719             } elsif ($opt eq 'cpuunits') {
4720                 $cgroup->change_cpu_shares($conf->{pending}->{$opt}, $defaults->{cpuunits});
4721             } elsif ($opt eq 'cpulimit') {
4722                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4723                 $cgroup->change_cpu_quota($cpulimit, 100000);
4724             } else {
4725                 die "skip\n";  # skip non-hot-pluggable options
4726             }
4727         };
4728         if (my $err = $@) {
4729             &$add_error($opt, $err) if $err ne "skip\n";
4730         } else {
4731             $conf->{$opt} = $value;
4732             delete $conf->{pending}->{$opt};
4733         }
4734     }
4735
4736     PVE::QemuConfig->write_config($vmid, $conf);
4737 }
4738
4739 sub try_deallocate_drive {
4740     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4741
4742     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4743         my $volid = $drive->{file};
4744         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4745             my $sid = PVE::Storage::parse_volume_id($volid);
4746             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4747
4748             # check if the disk is really unused
4749             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4750                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4751             PVE::Storage::vdisk_free($storecfg, $volid);
4752             return 1;
4753         } else {
4754             # If vm is not owner of this disk remove from config
4755             return 1;
4756         }
4757     }
4758
4759     return;
4760 }
4761
4762 sub vmconfig_delete_or_detach_drive {
4763     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4764
4765     my $drive = parse_drive($opt, $conf->{$opt});
4766
4767     my $rpcenv = PVE::RPCEnvironment::get();
4768     my $authuser = $rpcenv->get_user();
4769
4770     if ($force) {
4771         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4772         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4773     } else {
4774         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4775     }
4776 }
4777
4778
4779
4780 sub vmconfig_apply_pending {
4781     my ($vmid, $conf, $storecfg, $errors) = @_;
4782
4783     my $add_apply_error = sub {
4784         my ($opt, $msg) = @_;
4785         my $err_msg = "unable to apply pending change $opt : $msg";
4786         $errors->{$opt} = $err_msg;
4787         warn $err_msg;
4788     };
4789
4790     # cold plug
4791
4792     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4793     foreach my $opt (sort keys %$pending_delete_hash) {
4794         my $force = $pending_delete_hash->{$opt}->{force};
4795         eval {
4796             if ($opt =~ m/^unused/) {
4797                 die "internal error";
4798             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4799                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4800             }
4801         };
4802         if (my $err = $@) {
4803             $add_apply_error->($opt, $err);
4804         } else {
4805             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4806             delete $conf->{$opt};
4807         }
4808     }
4809
4810     PVE::QemuConfig->cleanup_pending($conf);
4811
4812     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4813         next if $opt eq 'delete'; # just to be sure
4814         eval {
4815             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4816                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4817             }
4818         };
4819         if (my $err = $@) {
4820             $add_apply_error->($opt, $err);
4821         } else {
4822             $conf->{$opt} = delete $conf->{pending}->{$opt};
4823         }
4824     }
4825
4826     # write all changes at once to avoid unnecessary i/o
4827     PVE::QemuConfig->write_config($vmid, $conf);
4828 }
4829
4830 sub vmconfig_update_net {
4831     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4832
4833     my $newnet = parse_net($value);
4834
4835     if ($conf->{$opt}) {
4836         my $oldnet = parse_net($conf->{$opt});
4837
4838         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4839             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4840             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4841             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4842
4843             # for non online change, we try to hot-unplug
4844             die "skip\n" if !$hotplug;
4845             vm_deviceunplug($vmid, $conf, $opt);
4846         } else {
4847
4848             die "internal error" if $opt !~ m/net(\d+)/;
4849             my $iface = "tap${vmid}i$1";
4850
4851             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4852                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4853                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4854                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4855                 PVE::Network::tap_unplug($iface);
4856
4857                 if ($have_sdn) {
4858                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4859                 } else {
4860                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4861                 }
4862             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4863                 # Rate can be applied on its own but any change above needs to
4864                 # include the rate in tap_plug since OVS resets everything.
4865                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4866             }
4867
4868             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4869                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4870             }
4871
4872             return 1;
4873         }
4874     }
4875
4876     if ($hotplug) {
4877         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4878     } else {
4879         die "skip\n";
4880     }
4881 }
4882
4883 sub vmconfig_update_disk {
4884     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4885
4886     my $drive = parse_drive($opt, $value);
4887
4888     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
4889         my $media = $drive->{media} || 'disk';
4890         my $oldmedia = $old_drive->{media} || 'disk';
4891         die "unable to change media type\n" if $media ne $oldmedia;
4892
4893         if (!drive_is_cdrom($old_drive)) {
4894
4895             if ($drive->{file} ne $old_drive->{file}) {
4896
4897                 die "skip\n" if !$hotplug;
4898
4899                 # unplug and register as unused
4900                 vm_deviceunplug($vmid, $conf, $opt);
4901                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4902
4903             } else {
4904                 # update existing disk
4905
4906                 # skip non hotpluggable value
4907                 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4908                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4909                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4910                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
4911                     safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
4912                     die "skip\n";
4913                 }
4914
4915                 # apply throttle
4916                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4917                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4918                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4919                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4920                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4921                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4922                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4923                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4924                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4925                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4926                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4927                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4928                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4929                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4930                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4931                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4932                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4933                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4934
4935                     qemu_block_set_io_throttle(
4936                         $vmid,"drive-$opt",
4937                         ($drive->{mbps} || 0)*1024*1024,
4938                         ($drive->{mbps_rd} || 0)*1024*1024,
4939                         ($drive->{mbps_wr} || 0)*1024*1024,
4940                          $drive->{iops} || 0,
4941                          $drive->{iops_rd} || 0,
4942                          $drive->{iops_wr} || 0,
4943                         ($drive->{mbps_max} || 0)*1024*1024,
4944                         ($drive->{mbps_rd_max} || 0)*1024*1024,
4945                         ($drive->{mbps_wr_max} || 0)*1024*1024,
4946                          $drive->{iops_max} || 0,
4947                          $drive->{iops_rd_max} || 0,
4948                          $drive->{iops_wr_max} || 0,
4949                          $drive->{bps_max_length} || 1,
4950                          $drive->{bps_rd_max_length} || 1,
4951                          $drive->{bps_wr_max_length} || 1,
4952                          $drive->{iops_max_length} || 1,
4953                          $drive->{iops_rd_max_length} || 1,
4954                          $drive->{iops_wr_max_length} || 1,
4955                     );
4956
4957                 }
4958
4959                 return 1;
4960             }
4961
4962         } else { # cdrom
4963
4964             if ($drive->{file} eq 'none') {
4965                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4966                 if (drive_is_cloudinit($old_drive)) {
4967                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4968                 }
4969             } else {
4970                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4971
4972                 # force eject if locked
4973                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4974
4975                 if ($path) {
4976                     mon_cmd($vmid, "blockdev-change-medium",
4977                         id => "$opt", filename => "$path");
4978                 }
4979             }
4980
4981             return 1;
4982         }
4983     }
4984
4985     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4986     # hotplug new disks
4987     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4988     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
4989 }
4990
4991 # called in locked context by incoming migration
4992 sub vm_migrate_get_nbd_disks {
4993     my ($storecfg, $conf, $replicated_volumes) = @_;
4994
4995     my $local_volumes = {};
4996     PVE::QemuConfig->foreach_volume($conf, sub {
4997         my ($ds, $drive) = @_;
4998
4999         return if drive_is_cdrom($drive);
5000
5001         my $volid = $drive->{file};
5002
5003         return if !$volid;
5004
5005         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5006
5007         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5008         return if $scfg->{shared};
5009
5010         # replicated disks re-use existing state via bitmap
5011         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5012         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
5013     });
5014     return $local_volumes;
5015 }
5016
5017 # called in locked context by incoming migration
5018 sub vm_migrate_alloc_nbd_disks {
5019     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5020
5021     my $format = undef;
5022
5023     my $nbd = {};
5024     foreach my $opt (sort keys %$source_volumes) {
5025         my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
5026
5027         if ($use_existing) {
5028             $nbd->{$opt}->{drivestr} = print_drive($drive);
5029             $nbd->{$opt}->{volid} = $volid;
5030             $nbd->{$opt}->{replicated} = 1;
5031             next;
5032         }
5033
5034         # If a remote storage is specified and the format of the original
5035         # volume is not available there, fall back to the default format.
5036         # Otherwise use the same format as the original.
5037         if (!$storagemap->{identity}) {
5038             $storeid = map_storage($storagemap, $storeid);
5039             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5040             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5041             my $fileFormat = qemu_img_format($scfg, $volname);
5042             $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
5043         } else {
5044             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5045             $format = qemu_img_format($scfg, $volname);
5046         }
5047
5048         my $size = $drive->{size} / 1024;
5049         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5050         my $newdrive = $drive;
5051         $newdrive->{format} = $format;
5052         $newdrive->{file} = $newvolid;
5053         my $drivestr = print_drive($newdrive);
5054         $nbd->{$opt}->{drivestr} = $drivestr;
5055         $nbd->{$opt}->{volid} = $newvolid;
5056     }
5057
5058     return $nbd;
5059 }
5060
5061 # see vm_start_nolock for parameters, additionally:
5062 # migrate_opts:
5063 #   storagemap = parsed storage map for allocating NBD disks
5064 sub vm_start {
5065     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5066
5067     return PVE::QemuConfig->lock_config($vmid, sub {
5068         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5069
5070         die "you can't start a vm if it's a template\n"
5071             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5072
5073         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5074         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5075
5076         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5077
5078         if ($has_backup_lock && $running) {
5079             # a backup is currently running, attempt to start the guest in the
5080             # existing QEMU instance
5081             return vm_resume($vmid);
5082         }
5083
5084         PVE::QemuConfig->check_lock($conf)
5085             if !($params->{skiplock} || $has_suspended_lock);
5086
5087         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5088
5089         die "VM $vmid already running\n" if $running;
5090
5091         if (my $storagemap = $migrate_opts->{storagemap}) {
5092             my $replicated = $migrate_opts->{replicated_volumes};
5093             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5094             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5095
5096             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5097                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5098             }
5099         }
5100
5101         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5102     });
5103 }
5104
5105
5106 # params:
5107 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5108 #   skiplock => 0/1, skip checking for config lock
5109 #   skiptemplate => 0/1, skip checking whether VM is template
5110 #   forcemachine => to force Qemu machine (rollback/migration)
5111 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5112 #   timeout => in seconds
5113 #   paused => start VM in paused state (backup)
5114 #   resume => resume from hibernation
5115 #   pbs-backing => {
5116 #      sata0 => {
5117 #         repository
5118 #         snapshot
5119 #         keyfile
5120 #         archive
5121 #      },
5122 #      virtio2 => ...
5123 #   }
5124 # migrate_opts:
5125 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5126 #   migratedfrom => source node
5127 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5128 #   network => CIDR of migration network
5129 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5130 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5131 #   replicated_volumes = which volids should be re-used with bitmaps for nbd migration
5132 sub vm_start_nolock {
5133     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5134
5135     my $statefile = $params->{statefile};
5136     my $resume = $params->{resume};
5137
5138     my $migratedfrom = $migrate_opts->{migratedfrom};
5139     my $migration_type = $migrate_opts->{type};
5140
5141     my $res = {};
5142
5143     # clean up leftover reboot request files
5144     eval { clear_reboot_request($vmid); };
5145     warn $@ if $@;
5146
5147     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5148         vmconfig_apply_pending($vmid, $conf, $storecfg);
5149         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5150     }
5151
5152     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5153
5154     my $defaults = load_defaults();
5155
5156     # set environment variable useful inside network script
5157     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5158
5159     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5160
5161     my $forcemachine = $params->{forcemachine};
5162     my $forcecpu = $params->{forcecpu};
5163     if ($resume) {
5164         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5165         $forcemachine = $conf->{runningmachine};
5166         $forcecpu = $conf->{runningcpu};
5167         print "Resuming suspended VM\n";
5168     }
5169
5170     my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid,
5171         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5172
5173     my $migration_ip;
5174     my $get_migration_ip = sub {
5175         my ($nodename) = @_;
5176
5177         return $migration_ip if defined($migration_ip);
5178
5179         my $cidr = $migrate_opts->{network};
5180
5181         if (!defined($cidr)) {
5182             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5183             $cidr = $dc_conf->{migration}->{network};
5184         }
5185
5186         if (defined($cidr)) {
5187             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5188
5189             die "could not get IP: no address configured on local " .
5190                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5191
5192             die "could not get IP: multiple addresses configured on local " .
5193                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5194
5195             $migration_ip = @$ips[0];
5196         }
5197
5198         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5199             if !defined($migration_ip);
5200
5201         return $migration_ip;
5202     };
5203
5204     my $migrate_uri;
5205     if ($statefile) {
5206         if ($statefile eq 'tcp') {
5207             my $localip = "localhost";
5208             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5209             my $nodename = nodename();
5210
5211             if (!defined($migration_type)) {
5212                 if (defined($datacenterconf->{migration}->{type})) {
5213                     $migration_type = $datacenterconf->{migration}->{type};
5214                 } else {
5215                     $migration_type = 'secure';
5216                 }
5217             }
5218
5219             if ($migration_type eq 'insecure') {
5220                 $localip = $get_migration_ip->($nodename);
5221                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5222             }
5223
5224             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5225             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5226             $migrate_uri = "tcp:${localip}:${migrate_port}";
5227             push @$cmd, '-incoming', $migrate_uri;
5228             push @$cmd, '-S';
5229
5230         } elsif ($statefile eq 'unix') {
5231             # should be default for secure migrations as a ssh TCP forward
5232             # tunnel is not deterministic reliable ready and fails regurarly
5233             # to set up in time, so use UNIX socket forwards
5234             my $socket_addr = "/run/qemu-server/$vmid.migrate";
5235             unlink $socket_addr;
5236
5237             $migrate_uri = "unix:$socket_addr";
5238
5239             push @$cmd, '-incoming', $migrate_uri;
5240             push @$cmd, '-S';
5241
5242         } elsif (-e $statefile) {
5243             push @$cmd, '-loadstate', $statefile;
5244         } else {
5245             my $statepath = PVE::Storage::path($storecfg, $statefile);
5246             push @$vollist, $statefile;
5247             push @$cmd, '-loadstate', $statepath;
5248         }
5249     } elsif ($params->{paused}) {
5250         push @$cmd, '-S';
5251     }
5252
5253     # host pci devices
5254     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5255       my $d = parse_hostpci($conf->{"hostpci$i"});
5256       next if !$d;
5257       my $pcidevices = $d->{pciid};
5258       foreach my $pcidevice (@$pcidevices) {
5259             my $pciid = $pcidevice->{id};
5260
5261             my $info = PVE::SysFSTools::pci_device_info("$pciid");
5262             die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5263             die "no pci device info for device '$pciid'\n" if !$info;
5264
5265             if ($d->{mdev}) {
5266                 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5267                 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5268             } else {
5269                 die "can't unbind/bind PCI group to VFIO '$pciid'\n"
5270                     if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5271                 die "can't reset PCI device '$pciid'\n"
5272                     if $info->{has_fl_reset} && !PVE::SysFSTools::pci_dev_reset($info);
5273             }
5274       }
5275     }
5276
5277     PVE::Storage::activate_volumes($storecfg, $vollist);
5278
5279     eval {
5280         run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5281             outfunc => sub {}, errfunc => sub {});
5282     };
5283     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5284     # timeout should be more than enough here...
5285     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5286
5287     my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5288                                               : $defaults->{cpuunits};
5289
5290     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5291     my %run_params = (
5292         timeout => $statefile ? undef : $start_timeout,
5293         umask => 0077,
5294         noerr => 1,
5295     );
5296
5297     # when migrating, prefix QEMU output so other side can pick up any
5298     # errors that might occur and show the user
5299     if ($migratedfrom) {
5300         $run_params{quiet} = 1;
5301         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5302     }
5303
5304     my %properties = (
5305         Slice => 'qemu.slice',
5306         KillMode => 'process',
5307         SendSIGKILL => 0,
5308         TimeoutStopUSec => ULONG_MAX, # infinity
5309     );
5310
5311     if (PVE::CGroup::cgroup_mode() == 2) {
5312         $cpuunits = 10000 if $cpuunits >= 10000; # else we get an error
5313         $properties{CPUWeight} = $cpuunits;
5314     } else {
5315         $properties{CPUShares} = $cpuunits;
5316     }
5317
5318     if (my $cpulimit = $conf->{cpulimit}) {
5319         $properties{CPUQuota} = int($cpulimit * 100);
5320     }
5321     $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5322
5323     my $run_qemu = sub {
5324         PVE::Tools::run_fork sub {
5325             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5326
5327             my $exitcode = run_command($cmd, %run_params);
5328             die "QEMU exited with code $exitcode\n" if $exitcode;
5329         };
5330     };
5331
5332     if ($conf->{hugepages}) {
5333
5334         my $code = sub {
5335             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5336             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5337
5338             PVE::QemuServer::Memory::hugepages_mount();
5339             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5340
5341             eval { $run_qemu->() };
5342             if (my $err = $@) {
5343                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5344                     if !$conf->{keephugepages};
5345                 die $err;
5346             }
5347
5348             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5349                 if !$conf->{keephugepages};
5350         };
5351         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5352
5353     } else {
5354         eval { $run_qemu->() };
5355     }
5356
5357     if (my $err = $@) {
5358         # deactivate volumes if start fails
5359         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5360         die "start failed: $err";
5361     }
5362
5363     print "migration listens on $migrate_uri\n" if $migrate_uri;
5364     $res->{migrate_uri} = $migrate_uri;
5365
5366     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5367         eval { mon_cmd($vmid, "cont"); };
5368         warn $@ if $@;
5369     }
5370
5371     #start nbd server for storage migration
5372     if (my $nbd = $migrate_opts->{nbd}) {
5373         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5374
5375         my $migrate_storage_uri;
5376         # nbd_protocol_version > 0 for unix socket support
5377         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5378             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5379             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5380             $migrate_storage_uri = "nbd:unix:$socket_path";
5381         } else {
5382             my $nodename = nodename();
5383             my $localip = $get_migration_ip->($nodename);
5384             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5385             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5386
5387             mon_cmd($vmid, "nbd-server-start", addr => {
5388                 type => 'inet',
5389                 data => {
5390                     host => "${localip}",
5391                     port => "${storage_migrate_port}",
5392                 },
5393             });
5394             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5395             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5396         }
5397
5398         $res->{migrate_storage_uri} = $migrate_storage_uri;
5399
5400         foreach my $opt (sort keys %$nbd) {
5401             my $drivestr = $nbd->{$opt}->{drivestr};
5402             my $volid = $nbd->{$opt}->{volid};
5403             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5404             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5405             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5406             print "re-using replicated volume: $opt - $volid\n"
5407                 if $nbd->{$opt}->{replicated};
5408
5409             $res->{drives}->{$opt} = $nbd->{$opt};
5410             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5411         }
5412     }
5413
5414     if ($migratedfrom) {
5415         eval {
5416             set_migration_caps($vmid);
5417         };
5418         warn $@ if $@;
5419
5420         if ($spice_port) {
5421             print "spice listens on port $spice_port\n";
5422             $res->{spice_port} = $spice_port;
5423             if ($migrate_opts->{spice_ticket}) {
5424                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5425                     $migrate_opts->{spice_ticket});
5426                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5427             }
5428         }
5429
5430     } else {
5431         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5432             if !$statefile && $conf->{balloon};
5433
5434         foreach my $opt (keys %$conf) {
5435             next if $opt !~  m/^net\d+$/;
5436             my $nicconf = parse_net($conf->{$opt});
5437             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5438         }
5439     }
5440
5441     mon_cmd($vmid, 'qom-set',
5442                 path => "machine/peripheral/balloon0",
5443                 property => "guest-stats-polling-interval",
5444                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5445
5446     if ($resume) {
5447         print "Resumed VM, removing state\n";
5448         if (my $vmstate = $conf->{vmstate}) {
5449             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5450             PVE::Storage::vdisk_free($storecfg, $vmstate);
5451         }
5452         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5453         PVE::QemuConfig->write_config($vmid, $conf);
5454     }
5455
5456     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5457
5458     return $res;
5459 }
5460
5461 sub vm_commandline {
5462     my ($storecfg, $vmid, $snapname) = @_;
5463
5464     my $conf = PVE::QemuConfig->load_config($vmid);
5465     my $forcemachine;
5466     my $forcecpu;
5467
5468     if ($snapname) {
5469         my $snapshot = $conf->{snapshots}->{$snapname};
5470         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5471
5472         # check for machine or CPU overrides in snapshot
5473         $forcemachine = $snapshot->{runningmachine};
5474         $forcecpu = $snapshot->{runningcpu};
5475
5476         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5477
5478         $conf = $snapshot;
5479     }
5480
5481     my $defaults = load_defaults();
5482
5483     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults,
5484         $forcemachine, $forcecpu);
5485
5486     return PVE::Tools::cmd2string($cmd);
5487 }
5488
5489 sub vm_reset {
5490     my ($vmid, $skiplock) = @_;
5491
5492     PVE::QemuConfig->lock_config($vmid, sub {
5493
5494         my $conf = PVE::QemuConfig->load_config($vmid);
5495
5496         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5497
5498         mon_cmd($vmid, "system_reset");
5499     });
5500 }
5501
5502 sub get_vm_volumes {
5503     my ($conf) = @_;
5504
5505     my $vollist = [];
5506     foreach_volid($conf, sub {
5507         my ($volid, $attr) = @_;
5508
5509         return if $volid =~ m|^/|;
5510
5511         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5512         return if !$sid;
5513
5514         push @$vollist, $volid;
5515     });
5516
5517     return $vollist;
5518 }
5519
5520 sub vm_stop_cleanup {
5521     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5522
5523     eval {
5524
5525         if (!$keepActive) {
5526             my $vollist = get_vm_volumes($conf);
5527             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5528         }
5529
5530         foreach my $ext (qw(mon qmp pid vnc qga)) {
5531             unlink "/var/run/qemu-server/${vmid}.$ext";
5532         }
5533
5534         if ($conf->{ivshmem}) {
5535             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5536             # just delete it for now, VMs which have this already open do not
5537             # are affected, but new VMs will get a separated one. If this
5538             # becomes an issue we either add some sort of ref-counting or just
5539             # add a "don't delete on stop" flag to the ivshmem format.
5540             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5541         }
5542
5543         foreach my $key (keys %$conf) {
5544             next if $key !~ m/^hostpci(\d+)$/;
5545             my $hostpciindex = $1;
5546             my $d = parse_hostpci($conf->{$key});
5547             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5548
5549             foreach my $pci (@{$d->{pciid}}) {
5550                 my $pciid = $pci->{id};
5551                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5552             }
5553         }
5554
5555         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5556     };
5557     warn $@ if $@; # avoid errors - just warn
5558 }
5559
5560 # call only in locked context
5561 sub _do_vm_stop {
5562     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5563
5564     my $pid = check_running($vmid, $nocheck);
5565     return if !$pid;
5566
5567     my $conf;
5568     if (!$nocheck) {
5569         $conf = PVE::QemuConfig->load_config($vmid);
5570         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5571         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5572             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5573             $timeout = $opts->{down} if $opts->{down};
5574         }
5575         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5576     }
5577
5578     eval {
5579         if ($shutdown) {
5580             if (defined($conf) && get_qga_key($conf, 'enabled')) {
5581                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5582             } else {
5583                 mon_cmd($vmid, "system_powerdown");
5584             }
5585         } else {
5586             mon_cmd($vmid, "quit");
5587         }
5588     };
5589     my $err = $@;
5590
5591     if (!$err) {
5592         $timeout = 60 if !defined($timeout);
5593
5594         my $count = 0;
5595         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5596             $count++;
5597             sleep 1;
5598         }
5599
5600         if ($count >= $timeout) {
5601             if ($force) {
5602                 warn "VM still running - terminating now with SIGTERM\n";
5603                 kill 15, $pid;
5604             } else {
5605                 die "VM quit/powerdown failed - got timeout\n";
5606             }
5607         } else {
5608             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5609             return;
5610         }
5611     } else {
5612        if (!check_running($vmid, $nocheck)) {
5613             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5614             return;
5615        }
5616        if ($force) {
5617             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5618             kill 15, $pid;
5619         } else {
5620             die "VM quit/powerdown failed\n";
5621         }
5622     }
5623
5624     # wait again
5625     $timeout = 10;
5626
5627     my $count = 0;
5628     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5629         $count++;
5630         sleep 1;
5631     }
5632
5633     if ($count >= $timeout) {
5634         warn "VM still running - terminating now with SIGKILL\n";
5635         kill 9, $pid;
5636         sleep 1;
5637     }
5638
5639     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5640 }
5641
5642 # Note: use $nocheck to skip tests if VM configuration file exists.
5643 # We need that when migration VMs to other nodes (files already moved)
5644 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5645 sub vm_stop {
5646     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5647
5648     $force = 1 if !defined($force) && !$shutdown;
5649
5650     if ($migratedfrom){
5651         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5652         kill 15, $pid if $pid;
5653         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5654         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5655         return;
5656     }
5657
5658     PVE::QemuConfig->lock_config($vmid, sub {
5659         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5660    });
5661 }
5662
5663 sub vm_reboot {
5664     my ($vmid, $timeout) = @_;
5665
5666     PVE::QemuConfig->lock_config($vmid, sub {
5667         eval {
5668
5669             # only reboot if running, as qmeventd starts it again on a stop event
5670             return if !check_running($vmid);
5671
5672             create_reboot_request($vmid);
5673
5674             my $storecfg = PVE::Storage::config();
5675             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5676
5677         };
5678         if (my $err = $@) {
5679             # avoid that the next normal shutdown will be confused for a reboot
5680             clear_reboot_request($vmid);
5681             die $err;
5682         }
5683    });
5684 }
5685
5686 # note: if using the statestorage parameter, the caller has to check privileges
5687 sub vm_suspend {
5688     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5689
5690     my $conf;
5691     my $path;
5692     my $storecfg;
5693     my $vmstate;
5694
5695     PVE::QemuConfig->lock_config($vmid, sub {
5696
5697         $conf = PVE::QemuConfig->load_config($vmid);
5698
5699         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5700         PVE::QemuConfig->check_lock($conf)
5701             if !($skiplock || $is_backing_up);
5702
5703         die "cannot suspend to disk during backup\n"
5704             if $is_backing_up && $includestate;
5705
5706         if ($includestate) {
5707             $conf->{lock} = 'suspending';
5708             my $date = strftime("%Y-%m-%d", localtime(time()));
5709             $storecfg = PVE::Storage::config();
5710             if (!$statestorage) {
5711                 $statestorage = find_vmstate_storage($conf, $storecfg);
5712                 # check permissions for the storage
5713                 my $rpcenv = PVE::RPCEnvironment::get();
5714                 if ($rpcenv->{type} ne 'cli') {
5715                     my $authuser = $rpcenv->get_user();
5716                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5717                 }
5718             }
5719
5720
5721             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
5722                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5723             $path = PVE::Storage::path($storecfg, $vmstate);
5724             PVE::QemuConfig->write_config($vmid, $conf);
5725         } else {
5726             mon_cmd($vmid, "stop");
5727         }
5728     });
5729
5730     if ($includestate) {
5731         # save vm state
5732         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5733
5734         eval {
5735             set_migration_caps($vmid, 1);
5736             mon_cmd($vmid, "savevm-start", statefile => $path);
5737             for(;;) {
5738                 my $state = mon_cmd($vmid, "query-savevm");
5739                 if (!$state->{status}) {
5740                     die "savevm not active\n";
5741                 } elsif ($state->{status} eq 'active') {
5742                     sleep(1);
5743                     next;
5744                 } elsif ($state->{status} eq 'completed') {
5745                     print "State saved, quitting\n";
5746                     last;
5747                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5748                     die "query-savevm failed with error '$state->{error}'\n"
5749                 } else {
5750                     die "query-savevm returned status '$state->{status}'\n";
5751                 }
5752             }
5753         };
5754         my $err = $@;
5755
5756         PVE::QemuConfig->lock_config($vmid, sub {
5757             $conf = PVE::QemuConfig->load_config($vmid);
5758             if ($err) {
5759                 # cleanup, but leave suspending lock, to indicate something went wrong
5760                 eval {
5761                     mon_cmd($vmid, "savevm-end");
5762                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5763                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5764                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
5765                     PVE::QemuConfig->write_config($vmid, $conf);
5766                 };
5767                 warn $@ if $@;
5768                 die $err;
5769             }
5770
5771             die "lock changed unexpectedly\n"
5772                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5773
5774             mon_cmd($vmid, "quit");
5775             $conf->{lock} = 'suspended';
5776             PVE::QemuConfig->write_config($vmid, $conf);
5777         });
5778     }
5779 }
5780
5781 sub vm_resume {
5782     my ($vmid, $skiplock, $nocheck) = @_;
5783
5784     PVE::QemuConfig->lock_config($vmid, sub {
5785         my $res = mon_cmd($vmid, 'query-status');
5786         my $resume_cmd = 'cont';
5787         my $reset = 0;
5788
5789         if ($res->{status}) {
5790             return if $res->{status} eq 'running'; # job done, go home
5791             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
5792             $reset = 1 if $res->{status} eq 'shutdown';
5793         }
5794
5795         if (!$nocheck) {
5796
5797             my $conf = PVE::QemuConfig->load_config($vmid);
5798
5799             PVE::QemuConfig->check_lock($conf)
5800                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5801         }
5802
5803         if ($reset) {
5804             # required if a VM shuts down during a backup and we get a resume
5805             # request before the backup finishes for example
5806             mon_cmd($vmid, "system_reset");
5807         }
5808         mon_cmd($vmid, $resume_cmd);
5809     });
5810 }
5811
5812 sub vm_sendkey {
5813     my ($vmid, $skiplock, $key) = @_;
5814
5815     PVE::QemuConfig->lock_config($vmid, sub {
5816
5817         my $conf = PVE::QemuConfig->load_config($vmid);
5818
5819         # there is no qmp command, so we use the human monitor command
5820         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5821         die $res if $res ne '';
5822     });
5823 }
5824
5825 # vzdump restore implementaion
5826
5827 sub tar_archive_read_firstfile {
5828     my $archive = shift;
5829
5830     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5831
5832     # try to detect archive type first
5833     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5834         die "unable to open file '$archive'\n";
5835     my $firstfile = <$fh>;
5836     kill 15, $pid;
5837     close $fh;
5838
5839     die "ERROR: archive contaions no data\n" if !$firstfile;
5840     chomp $firstfile;
5841
5842     return $firstfile;
5843 }
5844
5845 sub tar_restore_cleanup {
5846     my ($storecfg, $statfile) = @_;
5847
5848     print STDERR "starting cleanup\n";
5849
5850     if (my $fd = IO::File->new($statfile, "r")) {
5851         while (defined(my $line = <$fd>)) {
5852             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5853                 my $volid = $2;
5854                 eval {
5855                     if ($volid =~ m|^/|) {
5856                         unlink $volid || die 'unlink failed\n';
5857                     } else {
5858                         PVE::Storage::vdisk_free($storecfg, $volid);
5859                     }
5860                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5861                 };
5862                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5863             } else {
5864                 print STDERR "unable to parse line in statfile - $line";
5865             }
5866         }
5867         $fd->close();
5868     }
5869 }
5870
5871 sub restore_file_archive {
5872     my ($archive, $vmid, $user, $opts) = @_;
5873
5874     return restore_vma_archive($archive, $vmid, $user, $opts)
5875         if $archive eq '-';
5876
5877     my $info = PVE::Storage::archive_info($archive);
5878     my $format = $opts->{format} // $info->{format};
5879     my $comp = $info->{compression};
5880
5881     # try to detect archive format
5882     if ($format eq 'tar') {
5883         return restore_tar_archive($archive, $vmid, $user, $opts);
5884     } else {
5885         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5886     }
5887 }
5888
5889 # hepler to remove disks that will not be used after restore
5890 my $restore_cleanup_oldconf = sub {
5891     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
5892
5893     PVE::QemuConfig->foreach_volume($oldconf, sub {
5894         my ($ds, $drive) = @_;
5895
5896         return if drive_is_cdrom($drive, 1);
5897
5898         my $volid = $drive->{file};
5899         return if !$volid || $volid =~ m|^/|;
5900
5901         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
5902         return if !$path || !$owner || ($owner != $vmid);
5903
5904         # Note: only delete disk we want to restore
5905         # other volumes will become unused
5906         if ($virtdev_hash->{$ds}) {
5907             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
5908             if (my $err = $@) {
5909                 warn $err;
5910             }
5911         }
5912     });
5913
5914     # delete vmstate files, after the restore we have no snapshots anymore
5915     foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5916         my $snap = $oldconf->{snapshots}->{$snapname};
5917         if ($snap->{vmstate}) {
5918             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
5919             if (my $err = $@) {
5920                 warn $err;
5921             }
5922         }
5923     }
5924 };
5925
5926 # Helper to parse vzdump backup device hints
5927 #
5928 # $rpcenv: Environment, used to ckeck storage permissions
5929 # $user: User ID, to check storage permissions
5930 # $storecfg: Storage configuration
5931 # $fh: the file handle for reading the configuration
5932 # $devinfo: should contain device sizes for all backu-up'ed devices
5933 # $options: backup options (pool, default storage)
5934 #
5935 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
5936 my $parse_backup_hints = sub {
5937     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
5938
5939     my $virtdev_hash = {};
5940
5941     while (defined(my $line = <$fh>)) {
5942         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5943             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5944             die "archive does not contain data for drive '$virtdev'\n"
5945                 if !$devinfo->{$devname};
5946
5947             if (defined($options->{storage})) {
5948                 $storeid = $options->{storage} || 'local';
5949             } elsif (!$storeid) {
5950                 $storeid = 'local';
5951             }
5952             $format = 'raw' if !$format;
5953             $devinfo->{$devname}->{devname} = $devname;
5954             $devinfo->{$devname}->{virtdev} = $virtdev;
5955             $devinfo->{$devname}->{format} = $format;
5956             $devinfo->{$devname}->{storeid} = $storeid;
5957
5958             # check permission on storage
5959             my $pool = $options->{pool}; # todo: do we need that?
5960             if ($user ne 'root@pam') {
5961                 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5962             }
5963
5964             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5965         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
5966             my $virtdev = $1;
5967             my $drive = parse_drive($virtdev, $2);
5968             if (drive_is_cloudinit($drive)) {
5969                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5970                 $storeid = $options->{storage} if defined ($options->{storage});
5971                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5972                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
5973
5974                 $virtdev_hash->{$virtdev} = {
5975                     format => $format,
5976                     storeid => $storeid,
5977                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
5978                     is_cloudinit => 1,
5979                 };
5980             }
5981         }
5982     }
5983
5984     return $virtdev_hash;
5985 };
5986
5987 # Helper to allocate and activate all volumes required for a restore
5988 #
5989 # $storecfg: Storage configuration
5990 # $virtdev_hash: as returned by parse_backup_hints()
5991 #
5992 # Returns: { $virtdev => $volid }
5993 my $restore_allocate_devices = sub {
5994     my ($storecfg, $virtdev_hash, $vmid) = @_;
5995
5996     my $map = {};
5997     foreach my $virtdev (sort keys %$virtdev_hash) {
5998         my $d = $virtdev_hash->{$virtdev};
5999         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6000         my $storeid = $d->{storeid};
6001         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6002
6003         # test if requested format is supported
6004         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6005         my $supported = grep { $_ eq $d->{format} } @$validFormats;
6006         $d->{format} = $defFormat if !$supported;
6007
6008         my $name;
6009         if ($d->{is_cloudinit}) {
6010             $name = "vm-$vmid-cloudinit";
6011             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6012             if ($scfg->{path}) {
6013                 $name .= ".$d->{format}";
6014             }
6015         }
6016
6017         my $volid = PVE::Storage::vdisk_alloc(
6018             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6019
6020         print STDERR "new volume ID is '$volid'\n";
6021         $d->{volid} = $volid;
6022
6023         PVE::Storage::activate_volumes($storecfg, [$volid]);
6024
6025         $map->{$virtdev} = $volid;
6026     }
6027
6028     return $map;
6029 };
6030
6031 sub restore_update_config_line {
6032     my ($cookie, $map, $line, $unique) = @_;
6033
6034     return '' if $line =~ m/^\#qmdump\#/;
6035     return '' if $line =~ m/^\#vzdump\#/;
6036     return '' if $line =~ m/^lock:/;
6037     return '' if $line =~ m/^unused\d+:/;
6038     return '' if $line =~ m/^parent:/;
6039
6040     my $res = '';
6041
6042     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6043     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6044         # try to convert old 1.X settings
6045         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6046         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6047             my ($model, $macaddr) = split(/\=/, $devconfig);
6048             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6049             my $net = {
6050                 model => $model,
6051                 bridge => "vmbr$ind",
6052                 macaddr => $macaddr,
6053             };
6054             my $netstr = print_net($net);
6055
6056             $res .= "net$cookie->{netcount}: $netstr\n";
6057             $cookie->{netcount}++;
6058         }
6059     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6060         my ($id, $netstr) = ($1, $2);
6061         my $net = parse_net($netstr);
6062         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6063         $netstr = print_net($net);
6064         $res .= "$id: $netstr\n";
6065     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
6066         my $virtdev = $1;
6067         my $value = $3;
6068         my $di = parse_drive($virtdev, $value);
6069         if (defined($di->{backup}) && !$di->{backup}) {
6070             $res .= "#$line";
6071         } elsif ($map->{$virtdev}) {
6072             delete $di->{format}; # format can change on restore
6073             $di->{file} = $map->{$virtdev};
6074             $value = print_drive($di);
6075             $res .= "$virtdev: $value\n";
6076         } else {
6077             $res .= $line;
6078         }
6079     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6080         my $vmgenid = $1;
6081         if ($vmgenid ne '0') {
6082             # always generate a new vmgenid if there was a valid one setup
6083             $vmgenid = generate_uuid();
6084         }
6085         $res .= "vmgenid: $vmgenid\n";
6086     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6087         my ($uuid, $uuid_str);
6088         UUID::generate($uuid);
6089         UUID::unparse($uuid, $uuid_str);
6090         my $smbios1 = parse_smbios1($2);
6091         $smbios1->{uuid} = $uuid_str;
6092         $res .= $1.print_smbios1($smbios1)."\n";
6093     } else {
6094         $res .= $line;
6095     }
6096
6097     return $res;
6098 }
6099
6100 my $restore_deactivate_volumes = sub {
6101     my ($storecfg, $devinfo) = @_;
6102
6103     my $vollist = [];
6104     foreach my $devname (keys %$devinfo) {
6105         my $volid = $devinfo->{$devname}->{volid};
6106         push @$vollist, $volid if $volid;
6107     }
6108
6109     PVE::Storage::deactivate_volumes($storecfg, $vollist);
6110 };
6111
6112 my $restore_destroy_volumes = sub {
6113     my ($storecfg, $devinfo) = @_;
6114
6115     foreach my $devname (keys %$devinfo) {
6116         my $volid = $devinfo->{$devname}->{volid};
6117         next if !$volid;
6118         eval {
6119             if ($volid =~ m|^/|) {
6120                 unlink $volid || die 'unlink failed\n';
6121             } else {
6122                 PVE::Storage::vdisk_free($storecfg, $volid);
6123             }
6124             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6125         };
6126         print STDERR "unable to cleanup '$volid' - $@" if $@;
6127     }
6128 };
6129
6130 sub scan_volids {
6131     my ($cfg, $vmid) = @_;
6132
6133     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6134
6135     my $volid_hash = {};
6136     foreach my $storeid (keys %$info) {
6137         foreach my $item (@{$info->{$storeid}}) {
6138             next if !($item->{volid} && $item->{size});
6139             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6140             $volid_hash->{$item->{volid}} = $item;
6141         }
6142     }
6143
6144     return $volid_hash;
6145 }
6146
6147 sub update_disk_config {
6148     my ($vmid, $conf, $volid_hash) = @_;
6149
6150     my $changes;
6151     my $prefix = "VM $vmid";
6152
6153     # used and unused disks
6154     my $referenced = {};
6155
6156     # Note: it is allowed to define multiple storages with same path (alias), so
6157     # we need to check both 'volid' and real 'path' (two different volid can point
6158     # to the same path).
6159
6160     my $referencedpath = {};
6161
6162     # update size info
6163     PVE::QemuConfig->foreach_volume($conf, sub {
6164         my ($opt, $drive) = @_;
6165
6166         my $volid = $drive->{file};
6167         return if !$volid;
6168         my $volume = $volid_hash->{$volid};
6169
6170         # mark volid as "in-use" for next step
6171         $referenced->{$volid} = 1;
6172         if ($volume && (my $path = $volume->{path})) {
6173             $referencedpath->{$path} = 1;
6174         }
6175
6176         return if drive_is_cdrom($drive);
6177         return if !$volume;
6178
6179         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6180         if (defined($updated)) {
6181             $changes = 1;
6182             $conf->{$opt} = print_drive($updated);
6183             print "$prefix ($opt): $msg\n";
6184         }
6185     });
6186
6187     # remove 'unusedX' entry if volume is used
6188     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6189         my ($opt, $drive) = @_;
6190
6191         my $volid = $drive->{file};
6192         return if !$volid;
6193
6194         my $path;
6195         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6196         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6197             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6198             $changes = 1;
6199             delete $conf->{$opt};
6200         }
6201
6202         $referenced->{$volid} = 1;
6203         $referencedpath->{$path} = 1 if $path;
6204     });
6205
6206     foreach my $volid (sort keys %$volid_hash) {
6207         next if $volid =~ m/vm-$vmid-state-/;
6208         next if $referenced->{$volid};
6209         my $path = $volid_hash->{$volid}->{path};
6210         next if !$path; # just to be sure
6211         next if $referencedpath->{$path};
6212         $changes = 1;
6213         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6214         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6215         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6216     }
6217
6218     return $changes;
6219 }
6220
6221 sub rescan {
6222     my ($vmid, $nolock, $dryrun) = @_;
6223
6224     my $cfg = PVE::Storage::config();
6225
6226     print "rescan volumes...\n";
6227     my $volid_hash = scan_volids($cfg, $vmid);
6228
6229     my $updatefn =  sub {
6230         my ($vmid) = @_;
6231
6232         my $conf = PVE::QemuConfig->load_config($vmid);
6233
6234         PVE::QemuConfig->check_lock($conf);
6235
6236         my $vm_volids = {};
6237         foreach my $volid (keys %$volid_hash) {
6238             my $info = $volid_hash->{$volid};
6239             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6240         }
6241
6242         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6243
6244         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6245     };
6246
6247     if (defined($vmid)) {
6248         if ($nolock) {
6249             &$updatefn($vmid);
6250         } else {
6251             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6252         }
6253     } else {
6254         my $vmlist = config_list();
6255         foreach my $vmid (keys %$vmlist) {
6256             if ($nolock) {
6257                 &$updatefn($vmid);
6258             } else {
6259                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6260             }
6261         }
6262     }
6263 }
6264
6265 sub restore_proxmox_backup_archive {
6266     my ($archive, $vmid, $user, $options) = @_;
6267
6268     my $storecfg = PVE::Storage::config();
6269
6270     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6271     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6272
6273     my $fingerprint = $scfg->{fingerprint};
6274     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6275
6276     my $repo = PVE::PBSClient::get_repository($scfg);
6277
6278     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6279     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6280     local $ENV{PBS_PASSWORD} = $password;
6281     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6282
6283     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6284         PVE::Storage::parse_volname($storecfg, $archive);
6285
6286     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6287
6288     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6289
6290     my $tmpdir = "/var/tmp/vzdumptmp$$";
6291     rmtree $tmpdir;
6292     mkpath $tmpdir;
6293
6294     my $conffile = PVE::QemuConfig->config_file($vmid);
6295      # disable interrupts (always do cleanups)
6296     local $SIG{INT} =
6297         local $SIG{TERM} =
6298         local $SIG{QUIT} =
6299         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6300
6301     # Note: $oldconf is undef if VM does not exists
6302     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6303     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6304     my $new_conf_raw = '';
6305
6306     my $rpcenv = PVE::RPCEnvironment::get();
6307     my $devinfo = {};
6308
6309     eval {
6310         # enable interrupts
6311         local $SIG{INT} =
6312             local $SIG{TERM} =
6313             local $SIG{QUIT} =
6314             local $SIG{HUP} =
6315             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6316
6317         my $cfgfn = "$tmpdir/qemu-server.conf";
6318         my $firewall_config_fn = "$tmpdir/fw.conf";
6319         my $index_fn = "$tmpdir/index.json";
6320
6321         my $cmd = "restore";
6322
6323         my $param = [$pbs_backup_name, "index.json", $index_fn];
6324         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6325         my $index = PVE::Tools::file_get_contents($index_fn);
6326         $index = decode_json($index);
6327
6328         # print Dumper($index);
6329         foreach my $info (@{$index->{files}}) {
6330             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6331                 my $devname = $1;
6332                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6333                     $devinfo->{$devname}->{size} = $1;
6334                 } else {
6335                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6336                 }
6337             }
6338         }
6339
6340         my $is_qemu_server_backup = scalar(
6341             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6342         );
6343         if (!$is_qemu_server_backup) {
6344             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6345         }
6346         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6347
6348         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6349         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6350
6351         if ($has_firewall_config) {
6352             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6353             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6354
6355             my $pve_firewall_dir = '/etc/pve/firewall';
6356             mkdir $pve_firewall_dir; # make sure the dir exists
6357             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6358         }
6359
6360         my $fh = IO::File->new($cfgfn, "r") ||
6361             die "unable to read qemu-server.conf - $!\n";
6362
6363         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6364
6365         # fixme: rate limit?
6366
6367         # create empty/temp config
6368         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6369
6370         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6371
6372         # allocate volumes
6373         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6374
6375         foreach my $virtdev (sort keys %$virtdev_hash) {
6376             my $d = $virtdev_hash->{$virtdev};
6377             next if $d->{is_cloudinit}; # no need to restore cloudinit
6378
6379             # for live-restore we only want to preload the efidisk
6380             next if $options->{live} && $virtdev ne 'efidisk0';
6381
6382             my $volid = $d->{volid};
6383
6384             my $path = PVE::Storage::path($storecfg, $volid);
6385
6386             my $pbs_restore_cmd = [
6387                 '/usr/bin/pbs-restore',
6388                 '--repository', $repo,
6389                 $pbs_backup_name,
6390                 "$d->{devname}.img.fidx",
6391                 $path,
6392                 '--verbose',
6393                 ];
6394
6395             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6396             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
6397
6398             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6399                 push @$pbs_restore_cmd, '--skip-zero';
6400             }
6401
6402             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6403             print "restore proxmox backup image: $dbg_cmdstring\n";
6404             run_command($pbs_restore_cmd);
6405         }
6406
6407         $fh->seek(0, 0) || die "seek failed - $!\n";
6408
6409         my $cookie = { netcount => 0 };
6410         while (defined(my $line = <$fh>)) {
6411             $new_conf_raw .= restore_update_config_line(
6412                 $cookie,
6413                 $map,
6414                 $line,
6415                 $options->{unique},
6416             );
6417         }
6418
6419         $fh->close();
6420     };
6421     my $err = $@;
6422
6423     if ($err || !$options->{live}) {
6424         $restore_deactivate_volumes->($storecfg, $devinfo);
6425     }
6426
6427     rmtree $tmpdir;
6428
6429     if ($err) {
6430         $restore_destroy_volumes->($storecfg, $devinfo);
6431         die $err;
6432     }
6433
6434     if ($options->{live}) {
6435         # keep lock during live-restore
6436         $new_conf_raw .= "\nlock: create";
6437     }
6438
6439     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6440
6441     PVE::Cluster::cfs_update(); # make sure we read new file
6442
6443     eval { rescan($vmid, 1); };
6444     warn $@ if $@;
6445
6446     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
6447
6448     if ($options->{live}) {
6449         # enable interrupts
6450         local $SIG{INT} =
6451             local $SIG{TERM} =
6452             local $SIG{QUIT} =
6453             local $SIG{HUP} =
6454             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
6455
6456         my $conf = PVE::QemuConfig->load_config($vmid);
6457         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
6458
6459         delete $devinfo->{'drive-efidisk0'}; # this special drive is already restored before start
6460         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $repo, $keyfile, $pbs_backup_name);
6461
6462         PVE::QemuConfig->remove_lock($vmid, "create");
6463     }
6464 }
6465
6466 sub pbs_live_restore {
6467     my ($vmid, $conf, $storecfg, $restored_disks, $repo, $keyfile, $snap) = @_;
6468
6469     print "starting VM for live-restore\n";
6470     print "repository: '$repo', snapshot: '$snap'\n";
6471
6472     my $pbs_backing = {};
6473     for my $ds (keys %$restored_disks) {
6474         $ds =~ m/^drive-(.*)$/;
6475         my $confname = $1;
6476         $pbs_backing->{$confname} = {
6477             repository => $repo,
6478             snapshot => $snap,
6479             archive => "$ds.img.fidx",
6480         };
6481         $pbs_backing->{$confname}->{keyfile} = $keyfile if -e $keyfile;
6482
6483         my $drive = parse_drive($confname, $conf->{$confname});
6484         print "restoring '$ds' to '$drive->{file}'\n";
6485     }
6486
6487     my $drives_streamed = 0;
6488     eval {
6489         # make sure HA doesn't interrupt our restore by stopping the VM
6490         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
6491             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
6492         }
6493
6494         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
6495         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
6496         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
6497
6498         my $qmeventd_fd = register_qmeventd_handle($vmid);
6499
6500         # begin streaming, i.e. data copy from PBS to target disk for every vol,
6501         # this will effectively collapse the backing image chain consisting of
6502         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
6503         # removes itself once all backing images vanish with 'auto-remove=on')
6504         my $jobs = {};
6505         for my $ds (sort keys %$restored_disks) {
6506             my $job_id = "restore-$ds";
6507             mon_cmd($vmid, 'block-stream',
6508                 'job-id' => $job_id,
6509                 device => "$ds",
6510             );
6511             $jobs->{$job_id} = {};
6512         }
6513
6514         mon_cmd($vmid, 'cont');
6515         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
6516
6517         print "restore-drive jobs finished successfully, removing all tracking block devices"
6518             ." to disconnect from Proxmox Backup Server\n";
6519
6520         for my $ds (sort keys %$restored_disks) {
6521             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
6522         }
6523
6524         close($qmeventd_fd);
6525     };
6526
6527     my $err = $@;
6528
6529     if ($err) {
6530         warn "An error occured during live-restore: $err\n";
6531         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
6532         die "live-restore failed\n";
6533     }
6534 }
6535
6536 sub restore_vma_archive {
6537     my ($archive, $vmid, $user, $opts, $comp) = @_;
6538
6539     my $readfrom = $archive;
6540
6541     my $cfg = PVE::Storage::config();
6542     my $commands = [];
6543     my $bwlimit = $opts->{bwlimit};
6544
6545     my $dbg_cmdstring = '';
6546     my $add_pipe = sub {
6547         my ($cmd) = @_;
6548         push @$commands, $cmd;
6549         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6550         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6551         $readfrom = '-';
6552     };
6553
6554     my $input = undef;
6555     if ($archive eq '-') {
6556         $input = '<&STDIN';
6557     } else {
6558         # If we use a backup from a PVE defined storage we also consider that
6559         # storage's rate limit:
6560         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6561         if (defined($volid)) {
6562             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6563             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6564             if ($readlimit) {
6565                 print STDERR "applying read rate limit: $readlimit\n";
6566                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6567                 $add_pipe->($cstream);
6568             }
6569         }
6570     }
6571
6572     if ($comp) {
6573         my $info = PVE::Storage::decompressor_info('vma', $comp);
6574         my $cmd = $info->{decompressor};
6575         push @$cmd, $readfrom;
6576         $add_pipe->($cmd);
6577     }
6578
6579     my $tmpdir = "/var/tmp/vzdumptmp$$";
6580     rmtree $tmpdir;
6581
6582     # disable interrupts (always do cleanups)
6583     local $SIG{INT} =
6584         local $SIG{TERM} =
6585         local $SIG{QUIT} =
6586         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6587
6588     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6589     POSIX::mkfifo($mapfifo, 0600);
6590     my $fifofh;
6591     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
6592
6593     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6594
6595     my $oldtimeout;
6596     my $timeout = 5;
6597
6598     my $devinfo = {};
6599
6600     my $rpcenv = PVE::RPCEnvironment::get();
6601
6602     my $conffile = PVE::QemuConfig->config_file($vmid);
6603
6604     # Note: $oldconf is undef if VM does not exist
6605     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6606     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6607     my $new_conf_raw = '';
6608
6609     my %storage_limits;
6610
6611     my $print_devmap = sub {
6612         my $cfgfn = "$tmpdir/qemu-server.conf";
6613
6614         # we can read the config - that is already extracted
6615         my $fh = IO::File->new($cfgfn, "r") ||
6616             die "unable to read qemu-server.conf - $!\n";
6617
6618         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6619         if (-f $fwcfgfn) {
6620             my $pve_firewall_dir = '/etc/pve/firewall';
6621             mkdir $pve_firewall_dir; # make sure the dir exists
6622             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6623         }
6624
6625         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6626
6627         foreach my $info (values %{$virtdev_hash}) {
6628             my $storeid = $info->{storeid};
6629             next if defined($storage_limits{$storeid});
6630
6631             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
6632             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
6633             $storage_limits{$storeid} = $limit * 1024;
6634         }
6635
6636         foreach my $devname (keys %$devinfo) {
6637             die "found no device mapping information for device '$devname'\n"
6638                 if !$devinfo->{$devname}->{virtdev};
6639         }
6640
6641         # create empty/temp config
6642         if ($oldconf) {
6643             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6644             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6645         }
6646
6647         # allocate volumes
6648         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6649
6650         # print restore information to $fifofh
6651         foreach my $virtdev (sort keys %$virtdev_hash) {
6652             my $d = $virtdev_hash->{$virtdev};
6653             next if $d->{is_cloudinit}; # no need to restore cloudinit
6654
6655             my $storeid = $d->{storeid};
6656             my $volid = $d->{volid};
6657
6658             my $map_opts = '';
6659             if (my $limit = $storage_limits{$storeid}) {
6660                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6661             }
6662
6663             my $write_zeros = 1;
6664             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6665                 $write_zeros = 0;
6666             }
6667
6668             my $path = PVE::Storage::path($cfg, $volid);
6669
6670             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6671
6672             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6673         }
6674
6675         $fh->seek(0, 0) || die "seek failed - $!\n";
6676
6677         my $cookie = { netcount => 0 };
6678         while (defined(my $line = <$fh>)) {
6679             $new_conf_raw .= restore_update_config_line(
6680                 $cookie,
6681                 $map,
6682                 $line,
6683                 $opts->{unique},
6684             );
6685         }
6686
6687         $fh->close();
6688     };
6689
6690     eval {
6691         # enable interrupts
6692         local $SIG{INT} =
6693             local $SIG{TERM} =
6694             local $SIG{QUIT} =
6695             local $SIG{HUP} =
6696             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6697         local $SIG{ALRM} = sub { die "got timeout\n"; };
6698
6699         $oldtimeout = alarm($timeout);
6700
6701         my $parser = sub {
6702             my $line = shift;
6703
6704             print "$line\n";
6705
6706             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6707                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6708                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6709             } elsif ($line =~ m/^CTIME: /) {
6710                 # we correctly received the vma config, so we can disable
6711                 # the timeout now for disk allocation (set to 10 minutes, so
6712                 # that we always timeout if something goes wrong)
6713                 alarm(600);
6714                 &$print_devmap();
6715                 print $fifofh "done\n";
6716                 my $tmp = $oldtimeout || 0;
6717                 $oldtimeout = undef;
6718                 alarm($tmp);
6719                 close($fifofh);
6720                 $fifofh = undef;
6721             }
6722         };
6723
6724         print "restore vma archive: $dbg_cmdstring\n";
6725         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6726     };
6727     my $err = $@;
6728
6729     alarm($oldtimeout) if $oldtimeout;
6730
6731     $restore_deactivate_volumes->($cfg, $devinfo);
6732
6733     close($fifofh) if $fifofh;
6734     unlink $mapfifo;
6735     rmtree $tmpdir;
6736
6737     if ($err) {
6738         $restore_destroy_volumes->($cfg, $devinfo);
6739         die $err;
6740     }
6741
6742     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6743
6744     PVE::Cluster::cfs_update(); # make sure we read new file
6745
6746     eval { rescan($vmid, 1); };
6747     warn $@ if $@;
6748
6749     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
6750 }
6751
6752 sub restore_tar_archive {
6753     my ($archive, $vmid, $user, $opts) = @_;
6754
6755     if ($archive ne '-') {
6756         my $firstfile = tar_archive_read_firstfile($archive);
6757         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
6758             if $firstfile ne 'qemu-server.conf';
6759     }
6760
6761     my $storecfg = PVE::Storage::config();
6762
6763     # avoid zombie disks when restoring over an existing VM -> cleanup first
6764     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6765     # skiplock=1 because qmrestore has set the 'create' lock itself already
6766     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6767     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6768
6769     my $tocmd = "/usr/lib/qemu-server/qmextract";
6770
6771     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6772     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6773     $tocmd .= ' --prealloc' if $opts->{prealloc};
6774     $tocmd .= ' --info' if $opts->{info};
6775
6776     # tar option "xf" does not autodetect compression when read from STDIN,
6777     # so we pipe to zcat
6778     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6779         PVE::Tools::shellquote("--to-command=$tocmd");
6780
6781     my $tmpdir = "/var/tmp/vzdumptmp$$";
6782     mkpath $tmpdir;
6783
6784     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6785     local $ENV{VZDUMP_VMID} = $vmid;
6786     local $ENV{VZDUMP_USER} = $user;
6787
6788     my $conffile = PVE::QemuConfig->config_file($vmid);
6789     my $new_conf_raw = '';
6790
6791     # disable interrupts (always do cleanups)
6792     local $SIG{INT} =
6793         local $SIG{TERM} =
6794         local $SIG{QUIT} =
6795         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6796
6797     eval {
6798         # enable interrupts
6799         local $SIG{INT} =
6800             local $SIG{TERM} =
6801             local $SIG{QUIT} =
6802             local $SIG{HUP} =
6803             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6804
6805         if ($archive eq '-') {
6806             print "extracting archive from STDIN\n";
6807             run_command($cmd, input => "<&STDIN");
6808         } else {
6809             print "extracting archive '$archive'\n";
6810             run_command($cmd);
6811         }
6812
6813         return if $opts->{info};
6814
6815         # read new mapping
6816         my $map = {};
6817         my $statfile = "$tmpdir/qmrestore.stat";
6818         if (my $fd = IO::File->new($statfile, "r")) {
6819             while (defined (my $line = <$fd>)) {
6820                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6821                     $map->{$1} = $2 if $1;
6822                 } else {
6823                     print STDERR "unable to parse line in statfile - $line\n";
6824                 }
6825             }
6826             $fd->close();
6827         }
6828
6829         my $confsrc = "$tmpdir/qemu-server.conf";
6830
6831         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
6832
6833         my $cookie = { netcount => 0 };
6834         while (defined (my $line = <$srcfd>)) {
6835             $new_conf_raw .= restore_update_config_line(
6836                 $cookie,
6837                 $map,
6838                 $line,
6839                 $opts->{unique},
6840             );
6841         }
6842
6843         $srcfd->close();
6844     };
6845     if (my $err = $@) {
6846         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6847         die $err;
6848     }
6849
6850     rmtree $tmpdir;
6851
6852     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6853
6854     PVE::Cluster::cfs_update(); # make sure we read new file
6855
6856     eval { rescan($vmid, 1); };
6857     warn $@ if $@;
6858 };
6859
6860 sub foreach_storage_used_by_vm {
6861     my ($conf, $func) = @_;
6862
6863     my $sidhash = {};
6864
6865     PVE::QemuConfig->foreach_volume($conf, sub {
6866         my ($ds, $drive) = @_;
6867         return if drive_is_cdrom($drive);
6868
6869         my $volid = $drive->{file};
6870
6871         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6872         $sidhash->{$sid} = $sid if $sid;
6873     });
6874
6875     foreach my $sid (sort keys %$sidhash) {
6876         &$func($sid);
6877     }
6878 }
6879
6880 my $qemu_snap_storage = {
6881     rbd => 1,
6882 };
6883 sub do_snapshots_with_qemu {
6884     my ($storecfg, $volid) = @_;
6885
6886     my $storage_name = PVE::Storage::parse_volume_id($volid);
6887     my $scfg = $storecfg->{ids}->{$storage_name};
6888     die "could not find storage '$storage_name'\n" if !defined($scfg);
6889
6890     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6891         return 1;
6892     }
6893
6894     if ($volid =~ m/\.(qcow2|qed)$/){
6895         return 1;
6896     }
6897
6898     return;
6899 }
6900
6901 sub qga_check_running {
6902     my ($vmid, $nowarn) = @_;
6903
6904     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6905     if ($@) {
6906         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6907         return 0;
6908     }
6909     return 1;
6910 }
6911
6912 sub template_create {
6913     my ($vmid, $conf, $disk) = @_;
6914
6915     my $storecfg = PVE::Storage::config();
6916
6917     PVE::QemuConfig->foreach_volume($conf, sub {
6918         my ($ds, $drive) = @_;
6919
6920         return if drive_is_cdrom($drive);
6921         return if $disk && $ds ne $disk;
6922
6923         my $volid = $drive->{file};
6924         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6925
6926         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6927         $drive->{file} = $voliddst;
6928         $conf->{$ds} = print_drive($drive);
6929         PVE::QemuConfig->write_config($vmid, $conf);
6930     });
6931 }
6932
6933 sub convert_iscsi_path {
6934     my ($path) = @_;
6935
6936     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6937         my $portal = $1;
6938         my $target = $2;
6939         my $lun = $3;
6940
6941         my $initiator_name = get_initiator_name();
6942
6943         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6944                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6945     }
6946
6947     die "cannot convert iscsi path '$path', unkown format\n";
6948 }
6949
6950 sub qemu_img_convert {
6951     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6952
6953     my $storecfg = PVE::Storage::config();
6954     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6955     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6956
6957     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6958
6959     my $cachemode;
6960     my $src_path;
6961     my $src_is_iscsi = 0;
6962     my $src_format;
6963
6964     if ($src_storeid) {
6965         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6966         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6967         $src_format = qemu_img_format($src_scfg, $src_volname);
6968         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6969         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6970         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6971     } elsif (-f $src_volid) {
6972         $src_path = $src_volid;
6973         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6974             $src_format = $1;
6975         }
6976     }
6977
6978     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6979
6980     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6981     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6982     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6983     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6984
6985     my $cmd = [];
6986     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6987     push @$cmd, '-l', "snapshot.name=$snapname"
6988         if $snapname && $src_format && $src_format eq "qcow2";
6989     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6990     push @$cmd, '-T', $cachemode if defined($cachemode);
6991
6992     if ($src_is_iscsi) {
6993         push @$cmd, '--image-opts';
6994         $src_path = convert_iscsi_path($src_path);
6995     } elsif ($src_format) {
6996         push @$cmd, '-f', $src_format;
6997     }
6998
6999     if ($dst_is_iscsi) {
7000         push @$cmd, '--target-image-opts';
7001         $dst_path = convert_iscsi_path($dst_path);
7002     } else {
7003         push @$cmd, '-O', $dst_format;
7004     }
7005
7006     push @$cmd, $src_path;
7007
7008     if (!$dst_is_iscsi && $is_zero_initialized) {
7009         push @$cmd, "zeroinit:$dst_path";
7010     } else {
7011         push @$cmd, $dst_path;
7012     }
7013
7014     my $parser = sub {
7015         my $line = shift;
7016         if($line =~ m/\((\S+)\/100\%\)/){
7017             my $percent = $1;
7018             my $transferred = int($size * $percent / 100);
7019             my $total_h = render_bytes($size, 1);
7020             my $transferred_h = render_bytes($transferred, 1);
7021
7022             print "transferred $transferred_h of $total_h ($percent%)\n";
7023         }
7024
7025     };
7026
7027     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7028     my $err = $@;
7029     die "copy failed: $err" if $err;
7030 }
7031
7032 sub qemu_img_format {
7033     my ($scfg, $volname) = @_;
7034
7035     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7036         return $1;
7037     } else {
7038         return "raw";
7039     }
7040 }
7041
7042 sub qemu_drive_mirror {
7043     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7044
7045     $jobs = {} if !$jobs;
7046
7047     my $qemu_target;
7048     my $format;
7049     $jobs->{"drive-$drive"} = {};
7050
7051     if ($dst_volid =~ /^nbd:/) {
7052         $qemu_target = $dst_volid;
7053         $format = "nbd";
7054     } else {
7055         my $storecfg = PVE::Storage::config();
7056         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7057
7058         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7059
7060         $format = qemu_img_format($dst_scfg, $dst_volname);
7061
7062         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7063
7064         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7065     }
7066
7067     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7068     $opts->{format} = $format if $format;
7069
7070     if (defined($src_bitmap)) {
7071         $opts->{sync} = 'incremental';
7072         $opts->{bitmap} = $src_bitmap;
7073         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7074     }
7075
7076     if (defined($bwlimit)) {
7077         $opts->{speed} = $bwlimit * 1024;
7078         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7079     } else {
7080         print "drive mirror is starting for drive-$drive\n";
7081     }
7082
7083     # if a job already runs for this device we get an error, catch it for cleanup
7084     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7085     if (my $err = $@) {
7086         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7087         warn "$@\n" if $@;
7088         die "mirroring error: $err\n";
7089     }
7090
7091     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7092 }
7093
7094 # $completion can be either
7095 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7096 # 'cancel': wait until all jobs are ready, block-job-cancel them
7097 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7098 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7099 sub qemu_drive_mirror_monitor {
7100     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7101
7102     $completion //= 'complete';
7103     $op //= "mirror";
7104
7105     eval {
7106         my $err_complete = 0;
7107
7108         my $starttime = time ();
7109         while (1) {
7110             die "block job ('$op') timed out\n" if $err_complete > 300;
7111
7112             my $stats = mon_cmd($vmid, "query-block-jobs");
7113             my $ctime = time();
7114
7115             my $running_jobs = {};
7116             for my $stat (@$stats) {
7117                 next if $stat->{type} ne $op;
7118                 $running_jobs->{$stat->{device}} = $stat;
7119             }
7120
7121             my $readycounter = 0;
7122
7123             for my $job_id (sort keys %$jobs) {
7124                 my $job = $running_jobs->{$job_id};
7125
7126                 my $vanished = !defined($job);
7127                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7128                 if($complete || ($vanished && $completion eq 'auto')) {
7129                     print "$job_id: $op-job finished\n";
7130                     delete $jobs->{$job_id};
7131                     next;
7132                 }
7133
7134                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7135
7136                 my $busy = $job->{busy};
7137                 my $ready = $job->{ready};
7138                 if (my $total = $job->{len}) {
7139                     my $transferred = $job->{offset} || 0;
7140                     my $remaining = $total - $transferred;
7141                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7142
7143                     my $duration = $ctime - $starttime;
7144                     my $total_h = render_bytes($total, 1);
7145                     my $transferred_h = render_bytes($transferred, 1);
7146
7147                     my $status = sprintf(
7148                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7149                         render_duration($duration),
7150                     );
7151
7152                     if ($ready) {
7153                         if ($busy) {
7154                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7155                         } else {
7156                             $status .= ", ready";
7157                         }
7158                     }
7159                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7160                     $jobs->{$job_id}->{ready} = $ready;
7161                 }
7162
7163                 $readycounter++ if $job->{ready};
7164             }
7165
7166             last if scalar(keys %$jobs) == 0;
7167
7168             if ($readycounter == scalar(keys %$jobs)) {
7169                 print "all '$op' jobs are ready\n";
7170
7171                 # do the complete later (or has already been done)
7172                 last if $completion eq 'skip' || $completion eq 'auto';
7173
7174                 if ($vmiddst && $vmiddst != $vmid) {
7175                     my $agent_running = $qga && qga_check_running($vmid);
7176                     if ($agent_running) {
7177                         print "freeze filesystem\n";
7178                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7179                     } else {
7180                         print "suspend vm\n";
7181                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7182                     }
7183
7184                     # if we clone a disk for a new target vm, we don't switch the disk
7185                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7186
7187                     if ($agent_running) {
7188                         print "unfreeze filesystem\n";
7189                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7190                     } else {
7191                         print "resume vm\n";
7192                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
7193                     }
7194
7195                     last;
7196                 } else {
7197
7198                     for my $job_id (sort keys %$jobs) {
7199                         # try to switch the disk if source and destination are on the same guest
7200                         print "$job_id: Completing block job_id...\n";
7201
7202                         my $op;
7203                         if ($completion eq 'complete') {
7204                             $op = 'block-job-complete';
7205                         } elsif ($completion eq 'cancel') {
7206                             $op = 'block-job-cancel';
7207                         } else {
7208                             die "invalid completion value: $completion\n";
7209                         }
7210                         eval { mon_cmd($vmid, $op, device => $job_id) };
7211                         if ($@ =~ m/cannot be completed/) {
7212                             print "$job_id: block job cannot be completed, trying again.\n";
7213                             $err_complete++;
7214                         }else {
7215                             print "$job_id: Completed successfully.\n";
7216                             $jobs->{$job_id}->{complete} = 1;
7217                         }
7218                     }
7219                 }
7220             }
7221             sleep 1;
7222         }
7223     };
7224     my $err = $@;
7225
7226     if ($err) {
7227         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7228         die "block job ($op) error: $err";
7229     }
7230 }
7231
7232 sub qemu_blockjobs_cancel {
7233     my ($vmid, $jobs) = @_;
7234
7235     foreach my $job (keys %$jobs) {
7236         print "$job: Cancelling block job\n";
7237         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7238         $jobs->{$job}->{cancel} = 1;
7239     }
7240
7241     while (1) {
7242         my $stats = mon_cmd($vmid, "query-block-jobs");
7243
7244         my $running_jobs = {};
7245         foreach my $stat (@$stats) {
7246             $running_jobs->{$stat->{device}} = $stat;
7247         }
7248
7249         foreach my $job (keys %$jobs) {
7250
7251             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7252                 print "$job: Done.\n";
7253                 delete $jobs->{$job};
7254             }
7255         }
7256
7257         last if scalar(keys %$jobs) == 0;
7258
7259         sleep 1;
7260     }
7261 }
7262
7263 sub clone_disk {
7264     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7265         $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
7266
7267     my $newvolid;
7268
7269     if (!$full) {
7270         print "create linked clone of drive $drivename ($drive->{file})\n";
7271         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7272         push @$newvollist, $newvolid;
7273     } else {
7274
7275         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7276         $storeid = $storage if $storage;
7277
7278         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7279
7280         print "create full clone of drive $drivename ($drive->{file})\n";
7281         my $name = undef;
7282         my $size = undef;
7283         if (drive_is_cloudinit($drive)) {
7284             $name = "vm-$newvmid-cloudinit";
7285             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7286             if ($scfg->{path}) {
7287                 $name .= ".$dst_format";
7288             }
7289             $snapname = undef;
7290             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7291         } elsif ($drivename eq 'efidisk0') {
7292             $size = get_efivars_size($conf);
7293         } else {
7294             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
7295         }
7296         $newvolid = PVE::Storage::vdisk_alloc(
7297             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
7298         );
7299         push @$newvollist, $newvolid;
7300
7301         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7302
7303         if (drive_is_cloudinit($drive)) {
7304             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
7305             # if this is the case, we have to complete any block-jobs still there from
7306             # previous drive-mirrors
7307             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
7308                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
7309             }
7310             goto no_data_clone;
7311         }
7312
7313         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7314         if (!$running || $snapname) {
7315             # TODO: handle bwlimits
7316             if ($drivename eq 'efidisk0') {
7317                 # the relevant data on the efidisk may be smaller than the source
7318                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
7319                 # that is given by the OVMF_VARS.fd
7320                 my $src_path = PVE::Storage::path($storecfg, $drive->{file});
7321                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
7322
7323                 # better for Ceph if block size is not too small, see bug #3324
7324                 my $bs = 1024*1024;
7325
7326                 run_command(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=$bs", "osize=$size",
7327                     "if=$src_path", "of=$dst_path"]);
7328             } else {
7329                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7330             }
7331         } else {
7332
7333             my $kvmver = get_running_qemu_version ($vmid);
7334             if (!min_version($kvmver, 2, 7)) {
7335                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7336                     if $drive->{iothread};
7337             }
7338
7339             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7340                 $completion, $qga, $bwlimit);
7341         }
7342     }
7343
7344 no_data_clone:
7345     my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
7346
7347     my $disk = $drive;
7348     $disk->{format} = undef;
7349     $disk->{file} = $newvolid;
7350     $disk->{size} = $size if defined($size);
7351
7352     return $disk;
7353 }
7354
7355 sub get_running_qemu_version {
7356     my ($vmid) = @_;
7357     my $res = mon_cmd($vmid, "query-version");
7358     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7359 }
7360
7361 sub qemu_use_old_bios_files {
7362     my ($machine_type) = @_;
7363
7364     return if !$machine_type;
7365
7366     my $use_old_bios_files = undef;
7367
7368     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7369         $machine_type = $1;
7370         $use_old_bios_files = 1;
7371     } else {
7372         my $version = extract_version($machine_type, kvm_user_version());
7373         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7374         # load new efi bios files on migration. So this hack is required to allow
7375         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7376         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7377         $use_old_bios_files = !min_version($version, 2, 4);
7378     }
7379
7380     return ($use_old_bios_files, $machine_type);
7381 }
7382
7383 sub get_efivars_size {
7384     my ($conf) = @_;
7385     my $arch = get_vm_arch($conf);
7386     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7387     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
7388     return -s $ovmf_vars;
7389 }
7390
7391 sub update_efidisk_size {
7392     my ($conf) = @_;
7393
7394     return if !defined($conf->{efidisk0});
7395
7396     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
7397     $disk->{size} = get_efivars_size($conf);
7398     $conf->{efidisk0} = print_drive($disk);
7399
7400     return;
7401 }
7402
7403 sub create_efidisk($$$$$) {
7404     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7405
7406     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7407     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7408
7409     my $vars_size_b = -s $ovmf_vars;
7410     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7411     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7412     PVE::Storage::activate_volumes($storecfg, [$volid]);
7413
7414     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7415     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7416
7417     return ($volid, $size/1024);
7418 }
7419
7420 sub vm_iothreads_list {
7421     my ($vmid) = @_;
7422
7423     my $res = mon_cmd($vmid, 'query-iothreads');
7424
7425     my $iothreads = {};
7426     foreach my $iothread (@$res) {
7427         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7428     }
7429
7430     return $iothreads;
7431 }
7432
7433 sub scsihw_infos {
7434     my ($conf, $drive) = @_;
7435
7436     my $maxdev = 0;
7437
7438     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7439         $maxdev = 7;
7440     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7441         $maxdev = 1;
7442     } else {
7443         $maxdev = 256;
7444     }
7445
7446     my $controller = int($drive->{index} / $maxdev);
7447     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
7448         ? "virtioscsi"
7449         : "scsihw";
7450
7451     return ($maxdev, $controller, $controller_prefix);
7452 }
7453
7454 sub windows_version {
7455     my ($ostype) = @_;
7456
7457     return 0 if !$ostype;
7458
7459     my $winversion = 0;
7460
7461     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7462         $winversion = 5;
7463     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7464         $winversion = 6;
7465     } elsif ($ostype =~ m/^win(\d+)$/) {
7466         $winversion = $1;
7467     }
7468
7469     return $winversion;
7470 }
7471
7472 sub resolve_dst_disk_format {
7473         my ($storecfg, $storeid, $src_volname, $format) = @_;
7474         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7475
7476         if (!$format) {
7477             # if no target format is specified, use the source disk format as hint
7478             if ($src_volname) {
7479                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7480                 $format = qemu_img_format($scfg, $src_volname);
7481             } else {
7482                 return $defFormat;
7483             }
7484         }
7485
7486         # test if requested format is supported - else use default
7487         my $supported = grep { $_ eq $format } @$validFormats;
7488         $format = $defFormat if !$supported;
7489         return $format;
7490 }
7491
7492 # NOTE: if this logic changes, please update docs & possibly gui logic
7493 sub find_vmstate_storage {
7494     my ($conf, $storecfg) = @_;
7495
7496     # first, return storage from conf if set
7497     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7498
7499     my ($target, $shared, $local);
7500
7501     foreach_storage_used_by_vm($conf, sub {
7502         my ($sid) = @_;
7503         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7504         my $dst = $scfg->{shared} ? \$shared : \$local;
7505         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7506     });
7507
7508     # second, use shared storage where VM has at least one disk
7509     # third, use local storage where VM has at least one disk
7510     # fall back to local storage
7511     $target = $shared // $local // 'local';
7512
7513     return $target;
7514 }
7515
7516 sub generate_uuid {
7517     my ($uuid, $uuid_str);
7518     UUID::generate($uuid);
7519     UUID::unparse($uuid, $uuid_str);
7520     return $uuid_str;
7521 }
7522
7523 sub generate_smbios1_uuid {
7524     return "uuid=".generate_uuid();
7525 }
7526
7527 sub nbd_stop {
7528     my ($vmid) = @_;
7529
7530     mon_cmd($vmid, 'nbd-server-stop');
7531 }
7532
7533 sub create_reboot_request {
7534     my ($vmid) = @_;
7535     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7536         or die "failed to create reboot trigger file: $!\n";
7537     close($fh);
7538 }
7539
7540 sub clear_reboot_request {
7541     my ($vmid) = @_;
7542     my $path = "/run/qemu-server/$vmid.reboot";
7543     my $res = 0;
7544
7545     $res = unlink($path);
7546     die "could not remove reboot request for $vmid: $!"
7547         if !$res && $! != POSIX::ENOENT;
7548
7549     return $res;
7550 }
7551
7552 sub bootorder_from_legacy {
7553     my ($conf, $bootcfg) = @_;
7554
7555     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
7556     my $bootindex_hash = {};
7557     my $i = 1;
7558     foreach my $o (split(//, $boot)) {
7559         $bootindex_hash->{$o} = $i*100;
7560         $i++;
7561     }
7562
7563     my $bootorder = {};
7564
7565     PVE::QemuConfig->foreach_volume($conf, sub {
7566         my ($ds, $drive) = @_;
7567
7568         if (drive_is_cdrom ($drive, 1)) {
7569             if ($bootindex_hash->{d}) {
7570                 $bootorder->{$ds} = $bootindex_hash->{d};
7571                 $bootindex_hash->{d} += 1;
7572             }
7573         } elsif ($bootindex_hash->{c}) {
7574             $bootorder->{$ds} = $bootindex_hash->{c}
7575                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
7576             $bootindex_hash->{c} += 1;
7577         }
7578     });
7579
7580     if ($bootindex_hash->{n}) {
7581         for (my $i = 0; $i < $MAX_NETS; $i++) {
7582             my $netname = "net$i";
7583             next if !$conf->{$netname};
7584             $bootorder->{$netname} = $bootindex_hash->{n};
7585             $bootindex_hash->{n} += 1;
7586         }
7587     }
7588
7589     return $bootorder;
7590 }
7591
7592 # Generate default device list for 'boot: order=' property. Matches legacy
7593 # default boot order, but with explicit device names. This is important, since
7594 # the fallback for when neither 'order' nor the old format is specified relies
7595 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
7596 sub get_default_bootdevices {
7597     my ($conf) = @_;
7598
7599     my @ret = ();
7600
7601     # harddisk
7602     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
7603     push @ret, $first if $first;
7604
7605     # cdrom
7606     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
7607     push @ret, $first if $first;
7608
7609     # network
7610     for (my $i = 0; $i < $MAX_NETS; $i++) {
7611         my $netname = "net$i";
7612         next if !$conf->{$netname};
7613         push @ret, $netname;
7614         last;
7615     }
7616
7617     return \@ret;
7618 }
7619
7620 sub device_bootorder {
7621     my ($conf) = @_;
7622
7623     return bootorder_from_legacy($conf) if !defined($conf->{boot});
7624
7625     my $boot = parse_property_string($boot_fmt, $conf->{boot});
7626
7627     my $bootorder = {};
7628     if (!defined($boot) || $boot->{legacy}) {
7629         $bootorder = bootorder_from_legacy($conf, $boot);
7630     } elsif ($boot->{order}) {
7631         my $i = 100; # start at 100 to allow user to insert devices before us with -args
7632         for my $dev (PVE::Tools::split_list($boot->{order})) {
7633             $bootorder->{$dev} = $i++;
7634         }
7635     }
7636
7637     return $bootorder;
7638 }
7639
7640 sub register_qmeventd_handle {
7641     my ($vmid) = @_;
7642
7643     my $fh;
7644     my $peer = "/var/run/qmeventd.sock";
7645     my $count = 0;
7646
7647     for (;;) {
7648         $count++;
7649         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
7650         last if $fh;
7651         if ($! != EINTR && $! != EAGAIN) {
7652             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
7653         }
7654         if ($count > 4) {
7655             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
7656               . "after $count retries\n";
7657         }
7658         usleep(25000);
7659     }
7660
7661     # send handshake to mark VM as backing up
7662     print $fh to_json({vzdump => {vmid => "$vmid"}});
7663
7664     # return handle to be closed later when inhibit is no longer required
7665     return $fh;
7666 }
7667
7668 # bash completion helper
7669
7670 sub complete_backup_archives {
7671     my ($cmdname, $pname, $cvalue) = @_;
7672
7673     my $cfg = PVE::Storage::config();
7674
7675     my $storeid;
7676
7677     if ($cvalue =~ m/^([^:]+):/) {
7678         $storeid = $1;
7679     }
7680
7681     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7682
7683     my $res = [];
7684     foreach my $id (keys %$data) {
7685         foreach my $item (@{$data->{$id}}) {
7686             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
7687             push @$res, $item->{volid} if defined($item->{volid});
7688         }
7689     }
7690
7691     return $res;
7692 }
7693
7694 my $complete_vmid_full = sub {
7695     my ($running) = @_;
7696
7697     my $idlist = vmstatus();
7698
7699     my $res = [];
7700
7701     foreach my $id (keys %$idlist) {
7702         my $d = $idlist->{$id};
7703         if (defined($running)) {
7704             next if $d->{template};
7705             next if $running && $d->{status} ne 'running';
7706             next if !$running && $d->{status} eq 'running';
7707         }
7708         push @$res, $id;
7709
7710     }
7711     return $res;
7712 };
7713
7714 sub complete_vmid {
7715     return &$complete_vmid_full();
7716 }
7717
7718 sub complete_vmid_stopped {
7719     return &$complete_vmid_full(0);
7720 }
7721
7722 sub complete_vmid_running {
7723     return &$complete_vmid_full(1);
7724 }
7725
7726 sub complete_storage {
7727
7728     my $cfg = PVE::Storage::config();
7729     my $ids = $cfg->{ids};
7730
7731     my $res = [];
7732     foreach my $sid (keys %$ids) {
7733         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7734         next if !$ids->{$sid}->{content}->{images};
7735         push @$res, $sid;
7736     }
7737
7738     return $res;
7739 }
7740
7741 sub complete_migration_storage {
7742     my ($cmd, $param, $current_value, $all_args) = @_;
7743
7744     my $targetnode = @$all_args[1];
7745
7746     my $cfg = PVE::Storage::config();
7747     my $ids = $cfg->{ids};
7748
7749     my $res = [];
7750     foreach my $sid (keys %$ids) {
7751         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7752         next if !$ids->{$sid}->{content}->{images};
7753         push @$res, $sid;
7754     }
7755
7756     return $res;
7757 }
7758
7759 sub vm_is_paused {
7760     my ($vmid) = @_;
7761     my $qmpstatus = eval {
7762         PVE::QemuConfig::assert_config_exists_on_node($vmid);
7763         mon_cmd($vmid, "query-status");
7764     };
7765     warn "$@\n" if $@;
7766     return $qmpstatus && $qmpstatus->{status} eq "paused";
7767 }
7768
7769 sub check_volume_storage_type {
7770     my ($storecfg, $vol) = @_;
7771
7772     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
7773     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7774     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
7775
7776     die "storage '$storeid' does not support content-type '$vtype'\n"
7777         if !$scfg->{content}->{$vtype};
7778
7779     return 1;
7780 }
7781
7782 1;