PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::CGroup;
  31 use PVE::DataCenterConfig;
  32 use PVE::Exception qw(raise raise_param_exc);
  33 use PVE::Format qw(render_duration render_bytes);
  34 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  35 use PVE::INotify;
  36 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  37 use PVE::ProcFSTools;
  38 use PVE::PBSClient;
  39 use PVE::RPCEnvironment;
  40 use PVE::Storage;
  41 use PVE::SysFSTools;
  42 use PVE::Systemd;
  43 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  44
  45 use PVE::QMPClient;
  46 use PVE::QemuConfig;
  47 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  48 use PVE::QemuServer::Cloudinit;
  49 use PVE::QemuServer::CGroup;
  50 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  51 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  52 use PVE::QemuServer::Machine;
  53 use PVE::QemuServer::Memory;
  54 use PVE::QemuServer::Monitor qw(mon_cmd);
  55 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  56 use PVE::QemuServer::USB qw(parse_usb_device);
  57
  58 my $have_sdn;
  59 eval {
  60     require PVE::Network::SDN::Zones;
  61     $have_sdn = 1;
  62 };
  63
  64 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  65 my $OVMF = {
  66     x86_64 => [
  67         "$EDK2_FW_BASE/OVMF_CODE.fd",
  68         "$EDK2_FW_BASE/OVMF_VARS.fd"
  69     ],
  70     aarch64 => [
  71         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  72         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  73     ],
  74 };
  75
  76 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  77
  78 # Note about locking: we use flock on the config file protect
  79 # against concurent actions.
  80 # Aditionaly, we have a 'lock' setting in the config file. This
  81 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  82 # allowed when such lock is set. But you can ignore this kind of
  83 # lock with the --skiplock flag.
  84
  85 cfs_register_file('/qemu-server/',
  86                   \&parse_vm_config,
  87                   \&write_vm_config);
  88
  89 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  90     description => "Some command save/restore state from this location.",
  91     type => 'string',
  92     maxLength => 128,
  93     optional => 1,
  94 });
  95
  96 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  97         description => "Specifies the Qemu machine type.",
  98         type => 'string',
  99         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 100         maxLength => 40,
 101         optional => 1,
 102 });
 103
 104
 105 sub map_storage {
 106     my ($map, $source) = @_;
 107
 108     return $source if !defined($map);
 109
 110     return $map->{entries}->{$source}
 111         if $map->{entries} && defined($map->{entries}->{$source});
 112
 113     return $map->{default} if $map->{default};
 114
 115     # identity (fallback)
 116     return $source;
 117 }
 118
 119 PVE::JSONSchema::register_standard_option('pve-targetstorage', {
 120     description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
 121     type => 'string',
 122     format => 'storagepair-list',
 123     optional => 1,
 124 });
 125
 126 #no warnings 'redefine';
 127
 128 my $nodename_cache;
 129 sub nodename {
 130     $nodename_cache //= PVE::INotify::nodename();
 131     return $nodename_cache;
 132 }
 133
 134 my $watchdog_fmt = {
 135     model => {
 136         default_key => 1,
 137         type => 'string',
 138         enum => [qw(i6300esb ib700)],
 139         description => "Watchdog type to emulate.",
 140         default => 'i6300esb',
 141         optional => 1,
 142     },
 143     action => {
 144         type => 'string',
 145         enum => [qw(reset shutdown poweroff pause debug none)],
 146         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 147         optional => 1,
 148     },
 149 };
 150 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 151
 152 my $agent_fmt = {
 153     enabled => {
 154         description => "Enable/disable Qemu GuestAgent.",
 155         type => 'boolean',
 156         default => 0,
 157         default_key => 1,
 158     },
 159     fstrim_cloned_disks => {
 160         description => "Run fstrim after moving a disk or migrating the VM.",
 161         type => 'boolean',
 162         optional => 1,
 163         default => 0
 164     },
 165     type => {
 166         description => "Select the agent type",
 167         type => 'string',
 168         default => 'virtio',
 169         optional => 1,
 170         enum => [qw(virtio isa)],
 171     },
 172 };
 173
 174 my $vga_fmt = {
 175     type => {
 176         description => "Select the VGA type.",
 177         type => 'string',
 178         default => 'std',
 179         optional => 1,
 180         default_key => 1,
 181         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 182     },
 183     memory => {
 184         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 185         type => 'integer',
 186         optional => 1,
 187         minimum => 4,
 188         maximum => 512,
 189     },
 190 };
 191
 192 my $ivshmem_fmt = {
 193     size => {
 194         type => 'integer',
 195         minimum => 1,
 196         description => "The size of the file in MB.",
 197     },
 198     name => {
 199         type => 'string',
 200         pattern => '[a-zA-Z0-9\-]+',
 201         optional => 1,
 202         format_description => 'string',
 203         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 204     },
 205 };
 206
 207 my $audio_fmt = {
 208     device => {
 209         type => 'string',
 210         enum => [qw(ich9-intel-hda intel-hda AC97)],
 211         description =>  "Configure an audio device."
 212     },
 213     driver =>  {
 214         type => 'string',
 215         enum => ['spice', 'none'],
 216         default => 'spice',
 217         optional => 1,
 218         description => "Driver backend for the audio device."
 219     },
 220 };
 221
 222 my $spice_enhancements_fmt = {
 223     foldersharing => {
 224         type => 'boolean',
 225         optional => 1,
 226         default => '0',
 227         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 228     },
 229     videostreaming =>  {
 230         type => 'string',
 231         enum => ['off', 'all', 'filter'],
 232         default => 'off',
 233         optional => 1,
 234         description => "Enable video streaming. Uses compression for detected video streams."
 235     },
 236 };
 237
 238 my $rng_fmt = {
 239     source => {
 240         type => 'string',
 241         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 242         default_key => 1,
 243         description => "The file on the host to gather entropy from. In most"
 244                      . " cases /dev/urandom should be preferred over /dev/random"
 245                      . " to avoid entropy-starvation issues on the host. Using"
 246                      . " urandom does *not* decrease security in any meaningful"
 247                      . " way, as it's still seeded from real entropy, and the"
 248                      . " bytes provided will most likely be mixed with real"
 249                      . " entropy on the guest as well. /dev/hwrng can be used"
 250                      . " to pass through a hardware RNG from the host.",
 251     },
 252     max_bytes => {
 253         type => 'integer',
 254         description => "Maximum bytes of entropy injected into the guest every"
 255                      . " 'period' milliseconds. Prefer a lower value when using"
 256                      . " /dev/random as source. Use 0 to disable limiting"
 257                      . " (potentially dangerous!).",
 258         optional => 1,
 259
 260         # default is 1 KiB/s, provides enough entropy to the guest to avoid
 261         # boot-starvation issues (e.g. systemd etc...) while allowing no chance
 262         # of overwhelming the host, provided we're reading from /dev/urandom
 263         default => 1024,
 264     },
 265     period => {
 266         type => 'integer',
 267         description => "Every 'period' milliseconds the entropy-injection quota"
 268                      . " is reset, allowing the guest to retrieve another"
 269                      . " 'max_bytes' of entropy.",
 270         optional => 1,
 271         default => 1000,
 272     },
 273 };
 274
 275 my $confdesc = {
 276     onboot => {
 277         optional => 1,
 278         type => 'boolean',
 279         description => "Specifies whether a VM will be started during system bootup.",
 280         default => 0,
 281     },
 282     autostart => {
 283         optional => 1,
 284         type => 'boolean',
 285         description => "Automatic restart after crash (currently ignored).",
 286         default => 0,
 287     },
 288     hotplug => {
 289         optional => 1,
 290         type => 'string', format => 'pve-hotplug-features',
 291         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 292         default => 'network,disk,usb',
 293     },
 294     reboot => {
 295         optional => 1,
 296         type => 'boolean',
 297         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 298         default => 1,
 299     },
 300     lock => {
 301         optional => 1,
 302         type => 'string',
 303         description => "Lock/unlock the VM.",
 304         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 305     },
 306     cpulimit => {
 307         optional => 1,
 308         type => 'number',
 309         description => "Limit of CPU usage.",
 310         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 311         minimum => 0,
 312         maximum => 128,
 313         default => 0,
 314     },
 315     cpuunits => {
 316         optional => 1,
 317         type => 'integer',
 318         description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
 319         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
 320             ." The larger the number is, the more CPU time this VM gets. Number is relative to"
 321             ." weights of all the other running VMs.",
 322         minimum => 2,
 323         maximum => 262144,
 324         default => 'cgroup v1: 1024, cgroup v2: 100',
 325     },
 326     memory => {
 327         optional => 1,
 328         type => 'integer',
 329         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 330         minimum => 16,
 331         default => 512,
 332     },
 333     balloon => {
 334         optional => 1,
 335         type => 'integer',
 336         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 337         minimum => 0,
 338     },
 339     shares => {
 340         optional => 1,
 341         type => 'integer',
 342         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 343         minimum => 0,
 344         maximum => 50000,
 345         default => 1000,
 346     },
 347     keyboard => {
 348         optional => 1,
 349         type => 'string',
 350         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 351                        "It should not be necessary to set it.",
 352         enum => PVE::Tools::kvmkeymaplist(),
 353         default => undef,
 354     },
 355     name => {
 356         optional => 1,
 357         type => 'string', format => 'dns-name',
 358         description => "Set a name for the VM. Only used on the configuration web interface.",
 359     },
 360     scsihw => {
 361         optional => 1,
 362         type => 'string',
 363         description => "SCSI controller model",
 364         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 365         default => 'lsi',
 366     },
 367     description => {
 368         optional => 1,
 369         type => 'string',
 370         description => "Description for the VM. Shown in the web-interface VM's summary."
 371             ." This is saved as comment inside the configuration file.",
 372         maxLength => 1024 * 8,
 373     },
 374     ostype => {
 375         optional => 1,
 376         type => 'string',
 377         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 378         description => "Specify guest operating system.",
 379         verbose_description => <<EODESC,
 380 Specify guest operating system. This is used to enable special
 381 optimization/features for specific operating systems:
 382
 383 [horizontal]
 384 other;; unspecified OS
 385 wxp;; Microsoft Windows XP
 386 w2k;; Microsoft Windows 2000
 387 w2k3;; Microsoft Windows 2003
 388 w2k8;; Microsoft Windows 2008
 389 wvista;; Microsoft Windows Vista
 390 win7;; Microsoft Windows 7
 391 win8;; Microsoft Windows 8/2012/2012r2
 392 win10;; Microsoft Windows 10/2016/2019
 393 l24;; Linux 2.4 Kernel
 394 l26;; Linux 2.6 - 5.X Kernel
 395 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 396 EODESC
 397     },
 398     boot => {
 399         optional => 1,
 400         type => 'string', format => 'pve-qm-boot',
 401         description => "Specify guest boot order. Use with 'order=', usage with"
 402                      . " no key or 'legacy=' is deprecated.",
 403     },
 404     bootdisk => {
 405         optional => 1,
 406         type => 'string', format => 'pve-qm-bootdisk',
 407         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 408         pattern => '(ide|sata|scsi|virtio)\d+',
 409     },
 410     smp => {
 411         optional => 1,
 412         type => 'integer',
 413         description => "The number of CPUs. Please use option -sockets instead.",
 414         minimum => 1,
 415         default => 1,
 416     },
 417     sockets => {
 418         optional => 1,
 419         type => 'integer',
 420         description => "The number of CPU sockets.",
 421         minimum => 1,
 422         default => 1,
 423     },
 424     cores => {
 425         optional => 1,
 426         type => 'integer',
 427         description => "The number of cores per socket.",
 428         minimum => 1,
 429         default => 1,
 430     },
 431     numa => {
 432         optional => 1,
 433         type => 'boolean',
 434         description => "Enable/disable NUMA.",
 435         default => 0,
 436     },
 437     hugepages => {
 438         optional => 1,
 439         type => 'string',
 440         description => "Enable/disable hugepages memory.",
 441         enum => [qw(any 2 1024)],
 442     },
 443     keephugepages => {
 444         optional => 1,
 445         type => 'boolean',
 446         default => 0,
 447         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 448             ." after VM shutdown and can be used for subsequent starts.",
 449     },
 450     vcpus => {
 451         optional => 1,
 452         type => 'integer',
 453         description => "Number of hotplugged vcpus.",
 454         minimum => 1,
 455         default => 0,
 456     },
 457     acpi => {
 458         optional => 1,
 459         type => 'boolean',
 460         description => "Enable/disable ACPI.",
 461         default => 1,
 462     },
 463     agent => {
 464         optional => 1,
 465         description => "Enable/disable Qemu GuestAgent and its properties.",
 466         type => 'string',
 467         format => $agent_fmt,
 468     },
 469     kvm => {
 470         optional => 1,
 471         type => 'boolean',
 472         description => "Enable/disable KVM hardware virtualization.",
 473         default => 1,
 474     },
 475     tdf => {
 476         optional => 1,
 477         type => 'boolean',
 478         description => "Enable/disable time drift fix.",
 479         default => 0,
 480     },
 481     localtime => {
 482         optional => 1,
 483         type => 'boolean',
 484         description => "Set the real time clock to local time. This is enabled by default if ostype"
 485             ." indicates a Microsoft OS.",
 486     },
 487     freeze => {
 488         optional => 1,
 489         type => 'boolean',
 490         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 491     },
 492     vga => {
 493         optional => 1,
 494         type => 'string', format => $vga_fmt,
 495         description => "Configure the VGA hardware.",
 496         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 497             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 498             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 499             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 500             ." display server. For win* OS you can select how many independent displays you want,"
 501             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 502             ." using a serial device as terminal.",
 503     },
 504     watchdog => {
 505         optional => 1,
 506         type => 'string', format => 'pve-qm-watchdog',
 507         description => "Create a virtual hardware watchdog device.",
 508         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 509             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 510             ." else the watchdog will reset the guest (or execute the respective action specified)",
 511     },
 512     startdate => {
 513         optional => 1,
 514         type => 'string',
 515         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 516         description => "Set the initial date of the real time clock. Valid format for date are:"
 517             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 518         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 519         default => 'now',
 520     },
 521     startup =>  get_standard_option('pve-startup-order'),
 522     template => {
 523         optional => 1,
 524         type => 'boolean',
 525         description => "Enable/disable Template.",
 526         default => 0,
 527     },
 528     args => {
 529         optional => 1,
 530         type => 'string',
 531         description => "Arbitrary arguments passed to kvm.",
 532         verbose_description => <<EODESCR,
 533 Arbitrary arguments passed to kvm, for example:
 534
 535 args: -no-reboot -no-hpet
 536
 537 NOTE: this option is for experts only.
 538 EODESCR
 539     },
 540     tablet => {
 541         optional => 1,
 542         type => 'boolean',
 543         default => 1,
 544         description => "Enable/disable the USB tablet device.",
 545         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 546             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 547             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 548             ." may consider disabling this to save some context switches. This is turned off by"
 549             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 550     },
 551     migrate_speed => {
 552         optional => 1,
 553         type => 'integer',
 554         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 555         minimum => 0,
 556         default => 0,
 557     },
 558     migrate_downtime => {
 559         optional => 1,
 560         type => 'number',
 561         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 562         minimum => 0,
 563         default => 0.1,
 564     },
 565     cdrom => {
 566         optional => 1,
 567         type => 'string', format => 'pve-qm-ide',
 568         typetext => '<volume>',
 569         description => "This is an alias for option -ide2",
 570     },
 571     cpu => {
 572         optional => 1,
 573         description => "Emulated CPU type.",
 574         type => 'string',
 575         format => 'pve-vm-cpu-conf',
 576     },
 577     parent => get_standard_option('pve-snapshot-name', {
 578         optional => 1,
 579         description => "Parent snapshot name. This is used internally, and should not be modified.",
 580     }),
 581     snaptime => {
 582         optional => 1,
 583         description => "Timestamp for snapshots.",
 584         type => 'integer',
 585         minimum => 0,
 586     },
 587     vmstate => {
 588         optional => 1,
 589         type => 'string', format => 'pve-volume-id',
 590         description => "Reference to a volume which stores the VM state. This is used internally"
 591             ." for snapshots.",
 592     },
 593     vmstatestorage => get_standard_option('pve-storage-id', {
 594         description => "Default storage for VM state volumes/files.",
 595         optional => 1,
 596     }),
 597     runningmachine => get_standard_option('pve-qemu-machine', {
 598         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 599             ." for snapshots.",
 600     }),
 601     runningcpu => {
 602         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 603             ." internally for snapshots.",
 604         optional => 1,
 605         type => 'string',
 606         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 607         format_description => 'QEMU -cpu parameter'
 608     },
 609     machine => get_standard_option('pve-qemu-machine'),
 610     arch => {
 611         description => "Virtual processor architecture. Defaults to the host.",
 612         optional => 1,
 613         type => 'string',
 614         enum => [qw(x86_64 aarch64)],
 615     },
 616     smbios1 => {
 617         description => "Specify SMBIOS type 1 fields.",
 618         type => 'string', format => 'pve-qm-smbios1',
 619         maxLength => 512,
 620         optional => 1,
 621     },
 622     protection => {
 623         optional => 1,
 624         type => 'boolean',
 625         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 626             ." remove disk operations.",
 627         default => 0,
 628     },
 629     bios => {
 630         optional => 1,
 631         type => 'string',
 632         enum => [ qw(seabios ovmf) ],
 633         description => "Select BIOS implementation.",
 634         default => 'seabios',
 635     },
 636     vmgenid => {
 637         type => 'string',
 638         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 639         format_description => 'UUID',
 640         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 641             ." to disable explicitly.",
 642         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 643             ." value identifier to the guest OS. This allows to notify the guest operating system"
 644             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 645             ." execution or creation from a template). The guest operating system notices the"
 646             ." change, and is then able to react as appropriate by marking its copies of"
 647             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 648             ."Note that auto-creation only works when done through API/CLI create or update methods"
 649             .", but not when manually editing the config file.",
 650         default => "1 (autogenerated)",
 651         optional => 1,
 652     },
 653     hookscript => {
 654         type => 'string',
 655         format => 'pve-volume-id',
 656         optional => 1,
 657         description => "Script that will be executed during various steps in the vms lifetime.",
 658     },
 659     ivshmem => {
 660         type => 'string',
 661         format => $ivshmem_fmt,
 662         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 663             ." the host.",
 664         optional => 1,
 665     },
 666     audio0 => {
 667         type => 'string',
 668         format => $audio_fmt,
 669         description => "Configure a audio device, useful in combination with QXL/Spice.",
 670         optional => 1
 671     },
 672     spice_enhancements => {
 673         type => 'string',
 674         format => $spice_enhancements_fmt,
 675         description => "Configure additional enhancements for SPICE.",
 676         optional => 1
 677     },
 678     tags => {
 679         type => 'string', format => 'pve-tag-list',
 680         description => 'Tags of the VM. This is only meta information.',
 681         optional => 1,
 682     },
 683     rng0 => {
 684         type => 'string',
 685         format => $rng_fmt,
 686         description => "Configure a VirtIO-based Random Number Generator.",
 687         optional => 1,
 688     },
 689 };
 690
 691 my $cicustom_fmt = {
 692     meta => {
 693         type => 'string',
 694         optional => 1,
 695         description => 'Specify a custom file containing all meta data passed to the VM via"
 696             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 697         format => 'pve-volume-id',
 698         format_description => 'volume',
 699     },
 700     network => {
 701         type => 'string',
 702         optional => 1,
 703         description => 'Specify a custom file containing all network data passed to the VM via'
 704             .' cloud-init.',
 705         format => 'pve-volume-id',
 706         format_description => 'volume',
 707     },
 708     user => {
 709         type => 'string',
 710         optional => 1,
 711         description => 'Specify a custom file containing all user data passed to the VM via'
 712             .' cloud-init.',
 713         format => 'pve-volume-id',
 714         format_description => 'volume',
 715     },
 716 };
 717 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 718
 719 my $confdesc_cloudinit = {
 720     citype => {
 721         optional => 1,
 722         type => 'string',
 723         description => 'Specifies the cloud-init configuration format. The default depends on the'
 724             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 725             .' and `configdrive2` for windows.',
 726         enum => ['configdrive2', 'nocloud', 'opennebula'],
 727     },
 728     ciuser => {
 729         optional => 1,
 730         type => 'string',
 731         description => "cloud-init: User name to change ssh keys and password for instead of the"
 732             ." image's configured default user.",
 733     },
 734     cipassword => {
 735         optional => 1,
 736         type => 'string',
 737         description => 'cloud-init: Password to assign the user. Using this is generally not'
 738             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 739             .' support hashed passwords.',
 740     },
 741     cicustom => {
 742         optional => 1,
 743         type => 'string',
 744         description => 'cloud-init: Specify custom files to replace the automatically generated'
 745             .' ones at start.',
 746         format => 'pve-qm-cicustom',
 747     },
 748     searchdomain => {
 749         optional => 1,
 750         type => 'string',
 751         description => "cloud-init: Sets DNS search domains for a container. Create will'
 752             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 753             .' are set.",
 754     },
 755     nameserver => {
 756         optional => 1,
 757         type => 'string', format => 'address-list',
 758         description => "cloud-init: Sets DNS server IP address for a container. Create will'
 759             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 760             .' are set.",
 761     },
 762     sshkeys => {
 763         optional => 1,
 764         type => 'string',
 765         format => 'urlencoded',
 766         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 767     },
 768 };
 769
 770 # what about other qemu settings ?
 771 #cpu => 'string',
 772 #machine => 'string',
 773 #fda => 'file',
 774 #fdb => 'file',
 775 #mtdblock => 'file',
 776 #sd => 'file',
 777 #pflash => 'file',
 778 #snapshot => 'bool',
 779 #bootp => 'file',
 780 ##tftp => 'dir',
 781 ##smb => 'dir',
 782 #kernel => 'file',
 783 #append => 'string',
 784 #initrd => 'file',
 785 ##soundhw => 'string',
 786
 787 while (my ($k, $v) = each %$confdesc) {
 788     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 789 }
 790
 791 my $MAX_USB_DEVICES = 5;
 792 my $MAX_NETS = 32;
 793 my $MAX_SERIAL_PORTS = 4;
 794 my $MAX_PARALLEL_PORTS = 3;
 795 my $MAX_NUMA = 8;
 796
 797 my $numa_fmt = {
 798     cpus => {
 799         type => "string",
 800         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 801         description => "CPUs accessing this NUMA node.",
 802         format_description => "id[-id];...",
 803     },
 804     memory => {
 805         type => "number",
 806         description => "Amount of memory this NUMA node provides.",
 807         optional => 1,
 808     },
 809     hostnodes => {
 810         type => "string",
 811         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 812         description => "Host NUMA nodes to use.",
 813         format_description => "id[-id];...",
 814         optional => 1,
 815     },
 816     policy => {
 817         type => 'string',
 818         enum => [qw(preferred bind interleave)],
 819         description => "NUMA allocation policy.",
 820         optional => 1,
 821     },
 822 };
 823 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 824 my $numadesc = {
 825     optional => 1,
 826     type => 'string', format => $numa_fmt,
 827     description => "NUMA topology.",
 828 };
 829 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 830
 831 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 832     $confdesc->{"numa$i"} = $numadesc;
 833 }
 834
 835 my $nic_model_list = [
 836     'e1000',
 837     'e1000-82540em',
 838     'e1000-82544gc',
 839     'e1000-82545em',
 840     'e1000e',
 841     'i82551',
 842     'i82557b',
 843     'i82559er',
 844     'ne2k_isa',
 845     'ne2k_pci',
 846     'pcnet',
 847     'rtl8139',
 848     'virtio',
 849     'vmxnet3',
 850 ];
 851 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 852
 853 my $net_fmt_bridge_descr = <<__EOD__;
 854 Bridge to attach the network device to. The Proxmox VE standard bridge
 855 is called 'vmbr0'.
 856
 857 If you do not specify a bridge, we create a kvm user (NATed) network
 858 device, which provides DHCP and DNS services. The following addresses
 859 are used:
 860
 861  10.0.2.2   Gateway
 862  10.0.2.3   DNS Server
 863  10.0.2.4   SMB Server
 864
 865 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 866 __EOD__
 867
 868 my $net_fmt = {
 869     macaddr  => get_standard_option('mac-addr', {
 870         description => "MAC address. That address must be unique withing your network. This is"
 871             ." automatically generated if not specified.",
 872     }),
 873     model => {
 874         type => 'string',
 875         description => "Network Card Model. The 'virtio' model provides the best performance with"
 876             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 877             ." best to use 'e1000'.",
 878         enum => $nic_model_list,
 879         default_key => 1,
 880     },
 881     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 882     bridge => {
 883         type => 'string',
 884         description => $net_fmt_bridge_descr,
 885         format_description => 'bridge',
 886         pattern => '[-_.\w\d]+',
 887         optional => 1,
 888     },
 889     queues => {
 890         type => 'integer',
 891         minimum => 0, maximum => 16,
 892         description => 'Number of packet queues to be used on the device.',
 893         optional => 1,
 894     },
 895     rate => {
 896         type => 'number',
 897         minimum => 0,
 898         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 899         optional => 1,
 900     },
 901     tag => {
 902         type => 'integer',
 903         minimum => 1, maximum => 4094,
 904         description => 'VLAN tag to apply to packets on this interface.',
 905         optional => 1,
 906     },
 907     trunks => {
 908         type => 'string',
 909         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 910         description => 'VLAN trunks to pass through this interface.',
 911         format_description => 'vlanid[;vlanid...]',
 912         optional => 1,
 913     },
 914     firewall => {
 915         type => 'boolean',
 916         description => 'Whether this interface should be protected by the firewall.',
 917         optional => 1,
 918     },
 919     link_down => {
 920         type => 'boolean',
 921         description => 'Whether this interface should be disconnected (like pulling the plug).',
 922         optional => 1,
 923     },
 924     mtu => {
 925         type => 'integer',
 926         minimum => 1, maximum => 65520,
 927         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 928         optional => 1,
 929     },
 930 };
 931
 932 my $netdesc = {
 933     optional => 1,
 934     type => 'string', format => $net_fmt,
 935     description => "Specify network devices.",
 936 };
 937
 938 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 939
 940 my $ipconfig_fmt = {
 941     ip => {
 942         type => 'string',
 943         format => 'pve-ipv4-config',
 944         format_description => 'IPv4Format/CIDR',
 945         description => 'IPv4 address in CIDR format.',
 946         optional => 1,
 947         default => 'dhcp',
 948     },
 949     gw => {
 950         type => 'string',
 951         format => 'ipv4',
 952         format_description => 'GatewayIPv4',
 953         description => 'Default gateway for IPv4 traffic.',
 954         optional => 1,
 955         requires => 'ip',
 956     },
 957     ip6 => {
 958         type => 'string',
 959         format => 'pve-ipv6-config',
 960         format_description => 'IPv6Format/CIDR',
 961         description => 'IPv6 address in CIDR format.',
 962         optional => 1,
 963         default => 'dhcp',
 964     },
 965     gw6 => {
 966         type => 'string',
 967         format => 'ipv6',
 968         format_description => 'GatewayIPv6',
 969         description => 'Default gateway for IPv6 traffic.',
 970         optional => 1,
 971         requires => 'ip6',
 972     },
 973 };
 974 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 975 my $ipconfigdesc = {
 976     optional => 1,
 977     type => 'string', format => 'pve-qm-ipconfig',
 978     description => <<'EODESCR',
 979 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 980
 981 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 982
 983 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
 984 gateway should be provided.
 985 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
 986 cloud-init 19.4 or newer.
 987
 988 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
 989 dhcp on IPv4.
 990 EODESCR
 991 };
 992 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 993
 994 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 995     $confdesc->{"net$i"} = $netdesc;
 996     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 997 }
 998
 999 foreach my $key (keys %$confdesc_cloudinit) {
1000     $confdesc->{$key} = $confdesc_cloudinit->{$key};
1001 }
1002
1003 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1004 sub verify_volume_id_or_qm_path {
1005     my ($volid, $noerr) = @_;
1006
1007     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
1008         return $volid;
1009     }
1010
1011     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
1012     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1013     if ($@) {
1014         return if $noerr;
1015         die $@;
1016     }
1017     return $volid;
1018 }
1019
1020 my $usb_fmt = {
1021     host => {
1022         default_key => 1,
1023         type => 'string', format => 'pve-qm-usb-device',
1024         format_description => 'HOSTUSBDEVICE|spice',
1025         description => <<EODESCR,
1026 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1027
1028  'bus-port(.port)*' (decimal numbers) or
1029  'vendor_id:product_id' (hexadeciaml numbers) or
1030  'spice'
1031
1032 You can use the 'lsusb -t' command to list existing usb devices.
1033
1034 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1035 machines - use with special care.
1036
1037 The value 'spice' can be used to add a usb redirection devices for spice.
1038 EODESCR
1039     },
1040     usb3 => {
1041         optional => 1,
1042         type => 'boolean',
1043         description => "Specifies whether if given host option is a USB3 device or port.",
1044         default => 0,
1045     },
1046 };
1047
1048 my $usbdesc = {
1049     optional => 1,
1050     type => 'string', format => $usb_fmt,
1051     description => "Configure an USB device (n is 0 to 4).",
1052 };
1053 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1054
1055 my $serialdesc = {
1056         optional => 1,
1057         type => 'string',
1058         pattern => '(/dev/.+|socket)',
1059         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1060         verbose_description =>  <<EODESCR,
1061 Create a serial device inside the VM (n is 0 to 3), and pass through a
1062 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1063 host side (use 'qm terminal' to open a terminal connection).
1064
1065 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1066 use with special care.
1067
1068 CAUTION: Experimental! User reported problems with this option.
1069 EODESCR
1070 };
1071
1072 my $paralleldesc= {
1073         optional => 1,
1074         type => 'string',
1075         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1076         description =>  "Map host parallel devices (n is 0 to 2).",
1077         verbose_description =>  <<EODESCR,
1078 Map host parallel devices (n is 0 to 2).
1079
1080 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1081 machines - use with special care.
1082
1083 CAUTION: Experimental! User reported problems with this option.
1084 EODESCR
1085 };
1086
1087 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1088     $confdesc->{"parallel$i"} = $paralleldesc;
1089 }
1090
1091 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1092     $confdesc->{"serial$i"} = $serialdesc;
1093 }
1094
1095 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1096     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1097 }
1098
1099 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1100     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1101 }
1102
1103 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1104     $confdesc->{"usb$i"} = $usbdesc;
1105 }
1106
1107 my $boot_fmt = {
1108     legacy => {
1109         optional => 1,
1110         default_key => 1,
1111         type => 'string',
1112         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1113                      . " Deprecated, use 'order=' instead.",
1114         pattern => '[acdn]{1,4}',
1115         format_description => "[acdn]{1,4}",
1116
1117         # note: this is also the fallback if boot: is not given at all
1118         default => 'cdn',
1119     },
1120     order => {
1121         optional => 1,
1122         type => 'string',
1123         format => 'pve-qm-bootdev-list',
1124         format_description => "device[;device...]",
1125         description => <<EODESC,
1126 The guest will attempt to boot from devices in the order they appear here.
1127
1128 Disks, optical drives and passed-through storage USB devices will be directly
1129 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1130 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1131
1132 Note that only devices in this list will be marked as bootable and thus loaded
1133 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1134 (e.g. software-raid), you need to specify all of them here.
1135
1136 Overrides the deprecated 'legacy=[acdn]*' value when given.
1137 EODESC
1138     },
1139 };
1140 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1141
1142 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1143 sub verify_bootdev {
1144     my ($dev, $noerr) = @_;
1145
1146     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && $dev !~ m/^efidisk/;
1147
1148     my $check = sub {
1149         my ($base) = @_;
1150         return 0 if $dev !~ m/^$base\d+$/;
1151         return 0 if !$confdesc->{$dev};
1152         return 1;
1153     };
1154
1155     return $dev if $check->("net");
1156     return $dev if $check->("usb");
1157     return $dev if $check->("hostpci");
1158
1159     return if $noerr;
1160     die "invalid boot device '$dev'\n";
1161 }
1162
1163 sub print_bootorder {
1164     my ($devs) = @_;
1165     return "" if !@$devs;
1166     my $data = { order => join(';', @$devs) };
1167     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1168 }
1169
1170 my $kvm_api_version = 0;
1171
1172 sub kvm_version {
1173     return $kvm_api_version if $kvm_api_version;
1174
1175     open my $fh, '<', '/dev/kvm' or return;
1176
1177     # 0xae00 => KVM_GET_API_VERSION
1178     $kvm_api_version = ioctl($fh, 0xae00, 0);
1179     close($fh);
1180
1181     return $kvm_api_version;
1182 }
1183
1184 my $kvm_user_version = {};
1185 my $kvm_mtime = {};
1186
1187 sub kvm_user_version {
1188     my ($binary) = @_;
1189
1190     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1191     my $st = stat($binary);
1192
1193     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1194     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1195         $cachedmtime == $st->mtime;
1196
1197     $kvm_user_version->{$binary} = 'unknown';
1198     $kvm_mtime->{$binary} = $st->mtime;
1199
1200     my $code = sub {
1201         my $line = shift;
1202         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1203             $kvm_user_version->{$binary} = $2;
1204         }
1205     };
1206
1207     eval { run_command([$binary, '--version'], outfunc => $code); };
1208     warn $@ if $@;
1209
1210     return $kvm_user_version->{$binary};
1211
1212 }
1213 my sub extract_version {
1214     my ($machine_type, $version) = @_;
1215     $version = kvm_user_version() if !defined($version);
1216     PVE::QemuServer::Machine::extract_version($machine_type, $version)
1217 }
1218
1219 sub kernel_has_vhost_net {
1220     return -c '/dev/vhost-net';
1221 }
1222
1223 sub option_exists {
1224     my $key = shift;
1225     return defined($confdesc->{$key});
1226 }
1227
1228 my $cdrom_path;
1229 sub get_cdrom_path {
1230
1231     return  $cdrom_path if $cdrom_path;
1232
1233     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1234     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1235     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1236 }
1237
1238 sub get_iso_path {
1239     my ($storecfg, $vmid, $cdrom) = @_;
1240
1241     if ($cdrom eq 'cdrom') {
1242         return get_cdrom_path();
1243     } elsif ($cdrom eq 'none') {
1244         return '';
1245     } elsif ($cdrom =~ m|^/|) {
1246         return $cdrom;
1247     } else {
1248         return PVE::Storage::path($storecfg, $cdrom);
1249     }
1250 }
1251
1252 # try to convert old style file names to volume IDs
1253 sub filename_to_volume_id {
1254     my ($vmid, $file, $media) = @_;
1255
1256      if (!($file eq 'none' || $file eq 'cdrom' ||
1257           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1258
1259         return if $file =~ m|/|;
1260
1261         if ($media && $media eq 'cdrom') {
1262             $file = "local:iso/$file";
1263         } else {
1264             $file = "local:$vmid/$file";
1265         }
1266     }
1267
1268     return $file;
1269 }
1270
1271 sub verify_media_type {
1272     my ($opt, $vtype, $media) = @_;
1273
1274     return if !$media;
1275
1276     my $etype;
1277     if ($media eq 'disk') {
1278         $etype = 'images';
1279     } elsif ($media eq 'cdrom') {
1280         $etype = 'iso';
1281     } else {
1282         die "internal error";
1283     }
1284
1285     return if ($vtype eq $etype);
1286
1287     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1288 }
1289
1290 sub cleanup_drive_path {
1291     my ($opt, $storecfg, $drive) = @_;
1292
1293     # try to convert filesystem paths to volume IDs
1294
1295     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1296         ($drive->{file} !~ m|^/dev/.+|) &&
1297         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1298         ($drive->{file} !~ m/^\d+$/)) {
1299         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1300         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1301             if !$vtype;
1302         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1303         verify_media_type($opt, $vtype, $drive->{media});
1304         $drive->{file} = $volid;
1305     }
1306
1307     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1308 }
1309
1310 sub parse_hotplug_features {
1311     my ($data) = @_;
1312
1313     my $res = {};
1314
1315     return $res if $data eq '0';
1316
1317     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1318
1319     foreach my $feature (PVE::Tools::split_list($data)) {
1320         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1321             $res->{$1} = 1;
1322         } else {
1323             die "invalid hotplug feature '$feature'\n";
1324         }
1325     }
1326     return $res;
1327 }
1328
1329 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1330 sub pve_verify_hotplug_features {
1331     my ($value, $noerr) = @_;
1332
1333     return $value if parse_hotplug_features($value);
1334
1335     return if $noerr;
1336
1337     die "unable to parse hotplug option\n";
1338 }
1339
1340 sub scsi_inquiry {
1341     my($fh, $noerr) = @_;
1342
1343     my $SG_IO = 0x2285;
1344     my $SG_GET_VERSION_NUM = 0x2282;
1345
1346     my $versionbuf = "\x00" x 8;
1347     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1348     if (!$ret) {
1349         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1350         return;
1351     }
1352     my $version = unpack("I", $versionbuf);
1353     if ($version < 30000) {
1354         die "scsi generic interface too old\n"  if !$noerr;
1355         return;
1356     }
1357
1358     my $buf = "\x00" x 36;
1359     my $sensebuf = "\x00" x 8;
1360     my $cmd = pack("C x3 C x1", 0x12, 36);
1361
1362     # see /usr/include/scsi/sg.h
1363     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1364
1365     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1366                       length($sensebuf), 0, length($buf), $buf,
1367                       $cmd, $sensebuf, 6000);
1368
1369     $ret = ioctl($fh, $SG_IO, $packet);
1370     if (!$ret) {
1371         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1372         return;
1373     }
1374
1375     my @res = unpack($sg_io_hdr_t, $packet);
1376     if ($res[17] || $res[18]) {
1377         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1378         return;
1379     }
1380
1381     my $res = {};
1382     (my $byte0, my $byte1, $res->{vendor},
1383      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1384
1385     $res->{removable} = $byte1 & 128 ? 1 : 0;
1386     $res->{type} = $byte0 & 31;
1387
1388     return $res;
1389 }
1390
1391 sub path_is_scsi {
1392     my ($path) = @_;
1393
1394     my $fh = IO::File->new("+<$path") || return;
1395     my $res = scsi_inquiry($fh, 1);
1396     close($fh);
1397
1398     return $res;
1399 }
1400
1401 sub print_tabletdevice_full {
1402     my ($conf, $arch) = @_;
1403
1404     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1405
1406     # we use uhci for old VMs because tablet driver was buggy in older qemu
1407     my $usbbus;
1408     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1409         $usbbus = 'ehci';
1410     } else {
1411         $usbbus = 'uhci';
1412     }
1413
1414     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1415 }
1416
1417 sub print_keyboarddevice_full {
1418     my ($conf, $arch, $machine) = @_;
1419
1420     return if $arch ne 'aarch64';
1421
1422     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1423 }
1424
1425 my sub get_drive_id {
1426     my ($drive) = @_;
1427     return "$drive->{interface}$drive->{index}";
1428 }
1429
1430 sub print_drivedevice_full {
1431     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1432
1433     my $device = '';
1434     my $maxdev = 0;
1435
1436     my $drive_id = get_drive_id($drive);
1437     if ($drive->{interface} eq 'virtio') {
1438         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1439         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1440         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1441     } elsif ($drive->{interface} eq 'scsi') {
1442
1443         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1444         my $unit = $drive->{index} % $maxdev;
1445         my $devicetype = 'hd';
1446         my $path = '';
1447         if (drive_is_cdrom($drive)) {
1448             $devicetype = 'cd';
1449         } else {
1450             if ($drive->{file} =~ m|^/|) {
1451                 $path = $drive->{file};
1452                 if (my $info = path_is_scsi($path)) {
1453                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1454                         $devicetype = 'block';
1455                     } elsif ($info->{type} == 1) { # tape
1456                         $devicetype = 'generic';
1457                     }
1458                 }
1459             } else {
1460                  $path = PVE::Storage::path($storecfg, $drive->{file});
1461             }
1462
1463             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1464             my $version = extract_version($machine_type, kvm_user_version());
1465             if ($path =~ m/^iscsi\:\/\// &&
1466                !min_version($version, 4, 1)) {
1467                 $devicetype = 'generic';
1468             }
1469         }
1470
1471         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1472             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1473         } else {
1474             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1475                 .",lun=$drive->{index}";
1476         }
1477         $device .= ",drive=drive-$drive_id,id=$drive_id";
1478
1479         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1480             $device .= ",rotation_rate=1";
1481         }
1482         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1483
1484     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1485         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1486         my $controller = int($drive->{index} / $maxdev);
1487         my $unit = $drive->{index} % $maxdev;
1488         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1489
1490         $device = "ide-$devicetype";
1491         if ($drive->{interface} eq 'ide') {
1492             $device .= ",bus=ide.$controller,unit=$unit";
1493         } else {
1494             $device .= ",bus=ahci$controller.$unit";
1495         }
1496         $device .= ",drive=drive-$drive_id,id=$drive_id";
1497
1498         if ($devicetype eq 'hd') {
1499             if (my $model = $drive->{model}) {
1500                 $model = URI::Escape::uri_unescape($model);
1501                 $device .= ",model=$model";
1502             }
1503             if ($drive->{ssd}) {
1504                 $device .= ",rotation_rate=1";
1505             }
1506         }
1507         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1508     } elsif ($drive->{interface} eq 'usb') {
1509         die "implement me";
1510         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1511     } else {
1512         die "unsupported interface type";
1513     }
1514
1515     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1516
1517     if (my $serial = $drive->{serial}) {
1518         $serial = URI::Escape::uri_unescape($serial);
1519         $device .= ",serial=$serial";
1520     }
1521
1522
1523     return $device;
1524 }
1525
1526 sub get_initiator_name {
1527     my $initiator;
1528
1529     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1530     while (defined(my $line = <$fh>)) {
1531         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1532         $initiator = $1;
1533         last;
1534     }
1535     $fh->close();
1536
1537     return $initiator;
1538 }
1539
1540 sub print_drive_commandline_full {
1541     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1542
1543     my $path;
1544     my $volid = $drive->{file};
1545     my $format = $drive->{format};
1546     my $drive_id = get_drive_id($drive);
1547
1548     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1549     my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1550
1551     if (drive_is_cdrom($drive)) {
1552         $path = get_iso_path($storecfg, $vmid, $volid);
1553         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1554     } else {
1555         if ($storeid) {
1556             $path = PVE::Storage::path($storecfg, $volid);
1557             $format //= qemu_img_format($scfg, $volname);
1558         } else {
1559             $path = $volid;
1560             $format //= "raw";
1561         }
1562    }
1563
1564    my $is_rbd = $path =~ m/^rbd:/;
1565
1566     my $opts = '';
1567     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1568     foreach my $o (@qemu_drive_options) {
1569         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1570     }
1571
1572     # snapshot only accepts on|off
1573     if (defined($drive->{snapshot})) {
1574         my $v = $drive->{snapshot} ? 'on' : 'off';
1575         $opts .= ",snapshot=$v";
1576     }
1577
1578     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1579         my ($dir, $qmpname) = @$type;
1580         if (my $v = $drive->{"mbps$dir"}) {
1581             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1582         }
1583         if (my $v = $drive->{"mbps${dir}_max"}) {
1584             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1585         }
1586         if (my $v = $drive->{"bps${dir}_max_length"}) {
1587             $opts .= ",throttling.bps$qmpname-max-length=$v";
1588         }
1589         if (my $v = $drive->{"iops${dir}"}) {
1590             $opts .= ",throttling.iops$qmpname=$v";
1591         }
1592         if (my $v = $drive->{"iops${dir}_max"}) {
1593             $opts .= ",throttling.iops$qmpname-max=$v";
1594         }
1595         if (my $v = $drive->{"iops${dir}_max_length"}) {
1596             $opts .= ",throttling.iops$qmpname-max-length=$v";
1597         }
1598     }
1599
1600     if ($pbs_name) {
1601         $format = "rbd" if $is_rbd;
1602         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1603             if !$format;
1604         $opts .= ",format=alloc-track,file.driver=$format";
1605     } elsif ($format) {
1606         $opts .= ",format=$format";
1607     }
1608
1609     my $cache_direct = 0;
1610
1611     if (my $cache = $drive->{cache}) {
1612         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1613     } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1614         $opts .= ",cache=none";
1615         $cache_direct = 1;
1616     }
1617
1618     # io_uring with cache mode writeback or writethrough on krbd will hang...
1619     my $rbd_no_io_uring = $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1620
1621     # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1622     # sometimes, just plain disable...
1623     my $lvm_no_io_uring = $scfg && $scfg->{type} eq 'lvm';
1624
1625     if (!$drive->{aio}) {
1626         if ($io_uring && !$rbd_no_io_uring && !$lvm_no_io_uring) {
1627             # io_uring supports all cache modes
1628             $opts .= ",aio=io_uring";
1629         } else {
1630             # aio native works only with O_DIRECT
1631             if($cache_direct) {
1632                 $opts .= ",aio=native";
1633             } else {
1634                 $opts .= ",aio=threads";
1635             }
1636         }
1637     }
1638
1639     if (!drive_is_cdrom($drive)) {
1640         my $detectzeroes;
1641         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1642             $detectzeroes = 'off';
1643         } elsif ($drive->{discard}) {
1644             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1645         } else {
1646             # This used to be our default with discard not being specified:
1647             $detectzeroes = 'on';
1648         }
1649
1650         # note: 'detect-zeroes' works per blockdev and we want it to persist
1651         # after the alloc-track is removed, so put it on 'file' directly
1652         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1653         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1654     }
1655
1656     if ($pbs_name) {
1657         $opts .= ",backing=$pbs_name";
1658         $opts .= ",auto-remove=on";
1659     }
1660
1661     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1662     my $file_param = "file";
1663     if ($pbs_name) {
1664         # non-rbd drivers require the underlying file to be a seperate block
1665         # node, so add a second .file indirection
1666         $file_param .= ".file" if !$is_rbd;
1667         $file_param .= ".filename";
1668     }
1669     my $pathinfo = $path ? "$file_param=$path," : '';
1670
1671     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1672 }
1673
1674 sub print_pbs_blockdev {
1675     my ($pbs_conf, $pbs_name) = @_;
1676     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1677     $blockdev .= ",repository=$pbs_conf->{repository}";
1678     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1679     $blockdev .= ",archive=$pbs_conf->{archive}";
1680     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1681     return $blockdev;
1682 }
1683
1684 sub print_netdevice_full {
1685     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1686
1687     my $device = $net->{model};
1688     if ($net->{model} eq 'virtio') {
1689          $device = 'virtio-net-pci';
1690      };
1691
1692     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1693     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1694     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1695         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1696         # and out of each queue plus one config interrupt and control vector queue
1697         my $vectors = $net->{queues} * 2 + 2;
1698         $tmpstr .= ",vectors=$vectors,mq=on";
1699     }
1700     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1701
1702     if (my $mtu = $net->{mtu}) {
1703         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1704             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1705             if ($mtu == 1) {
1706                  $mtu = $bridge_mtu;
1707             } elsif ($mtu < 576) {
1708                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1709             } elsif ($mtu > $bridge_mtu) {
1710                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1711             }
1712             $tmpstr .= ",host_mtu=$mtu";
1713         } else {
1714             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1715         }
1716     }
1717
1718     if ($use_old_bios_files) {
1719         my $romfile;
1720         if ($device eq 'virtio-net-pci') {
1721             $romfile = 'pxe-virtio.rom';
1722         } elsif ($device eq 'e1000') {
1723             $romfile = 'pxe-e1000.rom';
1724         } elsif ($device eq 'e1000e') {
1725             $romfile = 'pxe-e1000e.rom';
1726         } elsif ($device eq 'ne2k') {
1727             $romfile = 'pxe-ne2k_pci.rom';
1728         } elsif ($device eq 'pcnet') {
1729             $romfile = 'pxe-pcnet.rom';
1730         } elsif ($device eq 'rtl8139') {
1731             $romfile = 'pxe-rtl8139.rom';
1732         }
1733         $tmpstr .= ",romfile=$romfile" if $romfile;
1734     }
1735
1736     return $tmpstr;
1737 }
1738
1739 sub print_netdev_full {
1740     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1741
1742     my $i = '';
1743     if ($netid =~ m/^net(\d+)$/) {
1744         $i = int($1);
1745     }
1746
1747     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1748
1749     my $ifname = "tap${vmid}i$i";
1750
1751     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1752     die "interface name '$ifname' is too long (max 15 character)\n"
1753         if length($ifname) >= 16;
1754
1755     my $vhostparam = '';
1756     if (is_native($arch)) {
1757         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1758     }
1759
1760     my $vmname = $conf->{name} || "vm$vmid";
1761
1762     my $netdev = "";
1763     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1764
1765     if ($net->{bridge}) {
1766         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1767             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1768     } else {
1769         $netdev = "type=user,id=$netid,hostname=$vmname";
1770     }
1771
1772     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1773
1774     return $netdev;
1775 }
1776
1777 my $vga_map = {
1778     'cirrus' => 'cirrus-vga',
1779     'std' => 'VGA',
1780     'vmware' => 'vmware-svga',
1781     'virtio' => 'virtio-vga',
1782 };
1783
1784 sub print_vga_device {
1785     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1786
1787     my $type = $vga_map->{$vga->{type}};
1788     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1789         $type = 'virtio-gpu';
1790     }
1791     my $vgamem_mb = $vga->{memory};
1792
1793     my $max_outputs = '';
1794     if ($qxlnum) {
1795         $type = $id ? 'qxl' : 'qxl-vga';
1796
1797         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1798             # set max outputs so linux can have up to 4 qxl displays with one device
1799             if (min_version($machine_version, 4, 1)) {
1800                 $max_outputs = ",max_outputs=4";
1801             }
1802         }
1803     }
1804
1805     die "no devicetype for $vga->{type}\n" if !$type;
1806
1807     my $memory = "";
1808     if ($vgamem_mb) {
1809         if ($vga->{type} eq 'virtio') {
1810             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1811             $memory = ",max_hostmem=$bytes";
1812         } elsif ($qxlnum) {
1813             # from https://www.spice-space.org/multiple-monitors.html
1814             $memory = ",vgamem_mb=$vga->{memory}";
1815             my $ram = $vgamem_mb * 4;
1816             my $vram = $vgamem_mb * 2;
1817             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1818         } else {
1819             $memory = ",vgamem_mb=$vga->{memory}";
1820         }
1821     } elsif ($qxlnum && $id) {
1822         $memory = ",ram_size=67108864,vram_size=33554432";
1823     }
1824
1825     my $edidoff = "";
1826     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1827         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1828     }
1829
1830     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1831     my $vgaid = "vga" . ($id // '');
1832     my $pciaddr;
1833
1834     if ($q35 && $vgaid eq 'vga') {
1835         # the first display uses pcie.0 bus on q35 machines
1836         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1837     } else {
1838         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1839     }
1840
1841     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1842 }
1843
1844 sub parse_number_sets {
1845     my ($set) = @_;
1846     my $res = [];
1847     foreach my $part (split(/;/, $set)) {
1848         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1849             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1850             push @$res, [ $1, $2 ];
1851         } else {
1852             die "invalid range: $part\n";
1853         }
1854     }
1855     return $res;
1856 }
1857
1858 sub parse_numa {
1859     my ($data) = @_;
1860
1861     my $res = parse_property_string($numa_fmt, $data);
1862     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1863     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1864     return $res;
1865 }
1866
1867 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1868 sub parse_net {
1869     my ($data) = @_;
1870
1871     my $res = eval { parse_property_string($net_fmt, $data) };
1872     if ($@) {
1873         warn $@;
1874         return;
1875     }
1876     if (!defined($res->{macaddr})) {
1877         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1878         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1879     }
1880     return $res;
1881 }
1882
1883 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1884 sub parse_ipconfig {
1885     my ($data) = @_;
1886
1887     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1888     if ($@) {
1889         warn $@;
1890         return;
1891     }
1892
1893     if ($res->{gw} && !$res->{ip}) {
1894         warn 'gateway specified without specifying an IP address';
1895         return;
1896     }
1897     if ($res->{gw6} && !$res->{ip6}) {
1898         warn 'IPv6 gateway specified without specifying an IPv6 address';
1899         return;
1900     }
1901     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1902         warn 'gateway specified together with DHCP';
1903         return;
1904     }
1905     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1906         # gw6 + auto/dhcp
1907         warn "IPv6 gateway specified together with $res->{ip6} address";
1908         return;
1909     }
1910
1911     if (!$res->{ip} && !$res->{ip6}) {
1912         return { ip => 'dhcp', ip6 => 'dhcp' };
1913     }
1914
1915     return $res;
1916 }
1917
1918 sub print_net {
1919     my $net = shift;
1920
1921     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1922 }
1923
1924 sub add_random_macs {
1925     my ($settings) = @_;
1926
1927     foreach my $opt (keys %$settings) {
1928         next if $opt !~ m/^net(\d+)$/;
1929         my $net = parse_net($settings->{$opt});
1930         next if !$net;
1931         $settings->{$opt} = print_net($net);
1932     }
1933 }
1934
1935 sub vm_is_volid_owner {
1936     my ($storecfg, $vmid, $volid) = @_;
1937
1938     if ($volid !~  m|^/|) {
1939         my ($path, $owner);
1940         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1941         if ($owner && ($owner == $vmid)) {
1942             return 1;
1943         }
1944     }
1945
1946     return;
1947 }
1948
1949 sub vmconfig_register_unused_drive {
1950     my ($storecfg, $vmid, $conf, $drive) = @_;
1951
1952     if (drive_is_cloudinit($drive)) {
1953         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1954         warn $@ if $@;
1955     } elsif (!drive_is_cdrom($drive)) {
1956         my $volid = $drive->{file};
1957         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1958             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1959         }
1960     }
1961 }
1962
1963 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1964 my $smbios1_fmt = {
1965     uuid => {
1966         type => 'string',
1967         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1968         format_description => 'UUID',
1969         description => "Set SMBIOS1 UUID.",
1970         optional => 1,
1971     },
1972     version => {
1973         type => 'string',
1974         pattern => '[A-Za-z0-9+\/]+={0,2}',
1975         format_description => 'Base64 encoded string',
1976         description => "Set SMBIOS1 version.",
1977         optional => 1,
1978     },
1979     serial => {
1980         type => 'string',
1981         pattern => '[A-Za-z0-9+\/]+={0,2}',
1982         format_description => 'Base64 encoded string',
1983         description => "Set SMBIOS1 serial number.",
1984         optional => 1,
1985     },
1986     manufacturer => {
1987         type => 'string',
1988         pattern => '[A-Za-z0-9+\/]+={0,2}',
1989         format_description => 'Base64 encoded string',
1990         description => "Set SMBIOS1 manufacturer.",
1991         optional => 1,
1992     },
1993     product => {
1994         type => 'string',
1995         pattern => '[A-Za-z0-9+\/]+={0,2}',
1996         format_description => 'Base64 encoded string',
1997         description => "Set SMBIOS1 product ID.",
1998         optional => 1,
1999     },
2000     sku => {
2001         type => 'string',
2002         pattern => '[A-Za-z0-9+\/]+={0,2}',
2003         format_description => 'Base64 encoded string',
2004         description => "Set SMBIOS1 SKU string.",
2005         optional => 1,
2006     },
2007     family => {
2008         type => 'string',
2009         pattern => '[A-Za-z0-9+\/]+={0,2}',
2010         format_description => 'Base64 encoded string',
2011         description => "Set SMBIOS1 family string.",
2012         optional => 1,
2013     },
2014     base64 => {
2015         type => 'boolean',
2016         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2017         optional => 1,
2018     },
2019 };
2020
2021 sub parse_smbios1 {
2022     my ($data) = @_;
2023
2024     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2025     warn $@ if $@;
2026     return $res;
2027 }
2028
2029 sub print_smbios1 {
2030     my ($smbios1) = @_;
2031     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2032 }
2033
2034 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2035
2036 sub parse_watchdog {
2037     my ($value) = @_;
2038
2039     return if !$value;
2040
2041     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2042     warn $@ if $@;
2043     return $res;
2044 }
2045
2046 sub parse_guest_agent {
2047     my ($conf) = @_;
2048
2049     return {} if !defined($conf->{agent});
2050
2051     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2052     warn $@ if $@;
2053
2054     # if the agent is disabled ignore the other potentially set properties
2055     return {} if !$res->{enabled};
2056     return $res;
2057 }
2058
2059 sub get_qga_key {
2060     my ($conf, $key) = @_;
2061     return undef if !defined($conf->{agent});
2062
2063     my $agent = parse_guest_agent($conf);
2064     return $agent->{$key};
2065 }
2066
2067 sub parse_vga {
2068     my ($value) = @_;
2069
2070     return {} if !$value;
2071     my $res = eval { parse_property_string($vga_fmt, $value) };
2072     warn $@ if $@;
2073     return $res;
2074 }
2075
2076 sub parse_rng {
2077     my ($value) = @_;
2078
2079     return if !$value;
2080
2081     my $res = eval { parse_property_string($rng_fmt, $value) };
2082     warn $@ if $@;
2083     return $res;
2084 }
2085
2086 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2087 sub verify_usb_device {
2088     my ($value, $noerr) = @_;
2089
2090     return $value if parse_usb_device($value);
2091
2092     return if $noerr;
2093
2094     die "unable to parse usb device\n";
2095 }
2096
2097 # add JSON properties for create and set function
2098 sub json_config_properties {
2099     my $prop = shift;
2100
2101     foreach my $opt (keys %$confdesc) {
2102         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' ||
2103             $opt eq 'runningmachine' || $opt eq 'runningcpu';
2104         $prop->{$opt} = $confdesc->{$opt};
2105     }
2106
2107     return $prop;
2108 }
2109
2110 # return copy of $confdesc_cloudinit to generate documentation
2111 sub cloudinit_config_properties {
2112
2113     return dclone($confdesc_cloudinit);
2114 }
2115
2116 sub check_type {
2117     my ($key, $value) = @_;
2118
2119     die "unknown setting '$key'\n" if !$confdesc->{$key};
2120
2121     my $type = $confdesc->{$key}->{type};
2122
2123     if (!defined($value)) {
2124         die "got undefined value\n";
2125     }
2126
2127     if ($value =~ m/[\n\r]/) {
2128         die "property contains a line feed\n";
2129     }
2130
2131     if ($type eq 'boolean') {
2132         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2133         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2134         die "type check ('boolean') failed - got '$value'\n";
2135     } elsif ($type eq 'integer') {
2136         return int($1) if $value =~ m/^(\d+)$/;
2137         die "type check ('integer') failed - got '$value'\n";
2138     } elsif ($type eq 'number') {
2139         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2140         die "type check ('number') failed - got '$value'\n";
2141     } elsif ($type eq 'string') {
2142         if (my $fmt = $confdesc->{$key}->{format}) {
2143             PVE::JSONSchema::check_format($fmt, $value);
2144             return $value;
2145         }
2146         $value =~ s/^\"(.*)\"$/$1/;
2147         return $value;
2148     } else {
2149         die "internal error"
2150     }
2151 }
2152
2153 sub destroy_vm {
2154     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2155
2156     my $conf = PVE::QemuConfig->load_config($vmid);
2157
2158     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2159
2160     if ($conf->{template}) {
2161         # check if any base image is still used by a linked clone
2162         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2163                 my ($ds, $drive) = @_;
2164                 return if drive_is_cdrom($drive);
2165
2166                 my $volid = $drive->{file};
2167                 return if !$volid || $volid =~ m|^/|;
2168
2169                 die "base volume '$volid' is still in use by linked cloned\n"
2170                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2171
2172         });
2173     }
2174
2175     my $remove_owned_drive = sub {
2176         my ($ds, $drive) = @_;
2177         return if drive_is_cdrom($drive, 1);
2178
2179         my $volid = $drive->{file};
2180         return if !$volid || $volid =~ m|^/|;
2181
2182         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2183         return if !$path || !$owner || ($owner != $vmid);
2184
2185         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2186         warn "Could not remove disk '$volid', check manually: $@" if $@;
2187     };
2188
2189     # only remove disks owned by this VM (referenced in the config)
2190     my $include_opts = {
2191         include_unused => 1,
2192         extra_keys => ['vmstate'],
2193     };
2194     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2195
2196     for my $snap (values %{$conf->{snapshots}}) {
2197         next if !defined($snap->{vmstate});
2198         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2199         next if !defined($drive);
2200         $remove_owned_drive->('vmstate', $drive);
2201     }
2202
2203     if ($purge_unreferenced) { # also remove unreferenced disk
2204         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2205         PVE::Storage::foreach_volid($vmdisks, sub {
2206             my ($volid, $sid, $volname, $d) = @_;
2207             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2208             warn $@ if $@;
2209         });
2210     }
2211
2212     if (defined $replacement_conf) {
2213         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2214     } else {
2215         PVE::QemuConfig->destroy_config($vmid);
2216     }
2217 }
2218
2219 sub parse_vm_config {
2220     my ($filename, $raw) = @_;
2221
2222     return if !defined($raw);
2223
2224     my $res = {
2225         digest => Digest::SHA::sha1_hex($raw),
2226         snapshots => {},
2227         pending => {},
2228     };
2229
2230     $filename =~ m|/qemu-server/(\d+)\.conf$|
2231         || die "got strange filename '$filename'";
2232
2233     my $vmid = $1;
2234
2235     my $conf = $res;
2236     my $descr;
2237     my $section = '';
2238
2239     my @lines = split(/\n/, $raw);
2240     foreach my $line (@lines) {
2241         next if $line =~ m/^\s*$/;
2242
2243         if ($line =~ m/^\[PENDING\]\s*$/i) {
2244             $section = 'pending';
2245             if (defined($descr)) {
2246                 $descr =~ s/\s+$//;
2247                 $conf->{description} = $descr;
2248             }
2249             $descr = undef;
2250             $conf = $res->{$section} = {};
2251             next;
2252
2253         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2254             $section = $1;
2255             if (defined($descr)) {
2256                 $descr =~ s/\s+$//;
2257                 $conf->{description} = $descr;
2258             }
2259             $descr = undef;
2260             $conf = $res->{snapshots}->{$section} = {};
2261             next;
2262         }
2263
2264         if ($line =~ m/^\#(.*)\s*$/) {
2265             $descr = '' if !defined($descr);
2266             $descr .= PVE::Tools::decode_text($1) . "\n";
2267             next;
2268         }
2269
2270         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2271             $descr = '' if !defined($descr);
2272             $descr .= PVE::Tools::decode_text($2);
2273         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2274             $conf->{snapstate} = $1;
2275         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2276             my $key = $1;
2277             my $value = $2;
2278             $conf->{$key} = $value;
2279         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2280             my $value = $1;
2281             if ($section eq 'pending') {
2282                 $conf->{delete} = $value; # we parse this later
2283             } else {
2284                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2285             }
2286         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2287             my $key = $1;
2288             my $value = $2;
2289             eval { $value = check_type($key, $value); };
2290             if ($@) {
2291                 warn "vm $vmid - unable to parse value of '$key' - $@";
2292             } else {
2293                 $key = 'ide2' if $key eq 'cdrom';
2294                 my $fmt = $confdesc->{$key}->{format};
2295                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2296                     my $v = parse_drive($key, $value);
2297                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2298                         $v->{file} = $volid;
2299                         $value = print_drive($v);
2300                     } else {
2301                         warn "vm $vmid - unable to parse value of '$key'\n";
2302                         next;
2303                     }
2304                 }
2305
2306                 $conf->{$key} = $value;
2307             }
2308         } else {
2309             warn "vm $vmid - unable to parse config: $line\n";
2310         }
2311     }
2312
2313     if (defined($descr)) {
2314         $descr =~ s/\s+$//;
2315         $conf->{description} = $descr;
2316     }
2317     delete $res->{snapstate}; # just to be sure
2318
2319     return $res;
2320 }
2321
2322 sub write_vm_config {
2323     my ($filename, $conf) = @_;
2324
2325     delete $conf->{snapstate}; # just to be sure
2326
2327     if ($conf->{cdrom}) {
2328         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2329         $conf->{ide2} = $conf->{cdrom};
2330         delete $conf->{cdrom};
2331     }
2332
2333     # we do not use 'smp' any longer
2334     if ($conf->{sockets}) {
2335         delete $conf->{smp};
2336     } elsif ($conf->{smp}) {
2337         $conf->{sockets} = $conf->{smp};
2338         delete $conf->{cores};
2339         delete $conf->{smp};
2340     }
2341
2342     my $used_volids = {};
2343
2344     my $cleanup_config = sub {
2345         my ($cref, $pending, $snapname) = @_;
2346
2347         foreach my $key (keys %$cref) {
2348             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2349                 $key eq 'snapstate' || $key eq 'pending';
2350             my $value = $cref->{$key};
2351             if ($key eq 'delete') {
2352                 die "propertry 'delete' is only allowed in [PENDING]\n"
2353                     if !$pending;
2354                 # fixme: check syntax?
2355                 next;
2356             }
2357             eval { $value = check_type($key, $value); };
2358             die "unable to parse value of '$key' - $@" if $@;
2359
2360             $cref->{$key} = $value;
2361
2362             if (!$snapname && is_valid_drivename($key)) {
2363                 my $drive = parse_drive($key, $value);
2364                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2365             }
2366         }
2367     };
2368
2369     &$cleanup_config($conf);
2370
2371     &$cleanup_config($conf->{pending}, 1);
2372
2373     foreach my $snapname (keys %{$conf->{snapshots}}) {
2374         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2375         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2376     }
2377
2378     # remove 'unusedX' settings if we re-add a volume
2379     foreach my $key (keys %$conf) {
2380         my $value = $conf->{$key};
2381         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2382             delete $conf->{$key};
2383         }
2384     }
2385
2386     my $generate_raw_config = sub {
2387         my ($conf, $pending) = @_;
2388
2389         my $raw = '';
2390
2391         # add description as comment to top of file
2392         if (defined(my $descr = $conf->{description})) {
2393             if ($descr) {
2394                 foreach my $cl (split(/\n/, $descr)) {
2395                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2396                 }
2397             } else {
2398                 $raw .= "#\n" if $pending;
2399             }
2400         }
2401
2402         foreach my $key (sort keys %$conf) {
2403             next if $key =~ /^(digest|description|pending|snapshots)$/;
2404             $raw .= "$key: $conf->{$key}\n";
2405         }
2406         return $raw;
2407     };
2408
2409     my $raw = &$generate_raw_config($conf);
2410
2411     if (scalar(keys %{$conf->{pending}})){
2412         $raw .= "\n[PENDING]\n";
2413         $raw .= &$generate_raw_config($conf->{pending}, 1);
2414     }
2415
2416     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2417         $raw .= "\n[$snapname]\n";
2418         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2419     }
2420
2421     return $raw;
2422 }
2423
2424 sub load_defaults {
2425
2426     my $res = {};
2427
2428     # we use static defaults from our JSON schema configuration
2429     foreach my $key (keys %$confdesc) {
2430         if (defined(my $default = $confdesc->{$key}->{default})) {
2431             $res->{$key} = $default;
2432         }
2433     }
2434
2435     return $res;
2436 }
2437
2438 sub config_list {
2439     my $vmlist = PVE::Cluster::get_vmlist();
2440     my $res = {};
2441     return $res if !$vmlist || !$vmlist->{ids};
2442     my $ids = $vmlist->{ids};
2443     my $nodename = nodename();
2444
2445     foreach my $vmid (keys %$ids) {
2446         my $d = $ids->{$vmid};
2447         next if !$d->{node} || $d->{node} ne $nodename;
2448         next if !$d->{type} || $d->{type} ne 'qemu';
2449         $res->{$vmid}->{exists} = 1;
2450     }
2451     return $res;
2452 }
2453
2454 # test if VM uses local resources (to prevent migration)
2455 sub check_local_resources {
2456     my ($conf, $noerr) = @_;
2457
2458     my @loc_res = ();
2459
2460     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2461     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2462
2463     push @loc_res, "ivshmem" if $conf->{ivshmem};
2464
2465     foreach my $k (keys %$conf) {
2466         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2467         # sockets are safe: they will recreated be on the target side post-migrate
2468         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2469         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2470     }
2471
2472     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2473
2474     return \@loc_res;
2475 }
2476
2477 # check if used storages are available on all nodes (use by migrate)
2478 sub check_storage_availability {
2479     my ($storecfg, $conf, $node) = @_;
2480
2481     PVE::QemuConfig->foreach_volume($conf, sub {
2482         my ($ds, $drive) = @_;
2483
2484         my $volid = $drive->{file};
2485         return if !$volid;
2486
2487         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2488         return if !$sid;
2489
2490         # check if storage is available on both nodes
2491         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2492         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2493
2494         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2495
2496         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2497             if !$scfg->{content}->{$vtype};
2498    });
2499 }
2500
2501 # list nodes where all VM images are available (used by has_feature API)
2502 sub shared_nodes {
2503     my ($conf, $storecfg) = @_;
2504
2505     my $nodelist = PVE::Cluster::get_nodelist();
2506     my $nodehash = { map { $_ => 1 } @$nodelist };
2507     my $nodename = nodename();
2508
2509     PVE::QemuConfig->foreach_volume($conf, sub {
2510         my ($ds, $drive) = @_;
2511
2512         my $volid = $drive->{file};
2513         return if !$volid;
2514
2515         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2516         if ($storeid) {
2517             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2518             if ($scfg->{disable}) {
2519                 $nodehash = {};
2520             } elsif (my $avail = $scfg->{nodes}) {
2521                 foreach my $node (keys %$nodehash) {
2522                     delete $nodehash->{$node} if !$avail->{$node};
2523                 }
2524             } elsif (!$scfg->{shared}) {
2525                 foreach my $node (keys %$nodehash) {
2526                     delete $nodehash->{$node} if $node ne $nodename
2527                 }
2528             }
2529         }
2530     });
2531
2532     return $nodehash
2533 }
2534
2535 sub check_local_storage_availability {
2536     my ($conf, $storecfg) = @_;
2537
2538     my $nodelist = PVE::Cluster::get_nodelist();
2539     my $nodehash = { map { $_ => {} } @$nodelist };
2540
2541     PVE::QemuConfig->foreach_volume($conf, sub {
2542         my ($ds, $drive) = @_;
2543
2544         my $volid = $drive->{file};
2545         return if !$volid;
2546
2547         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2548         if ($storeid) {
2549             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2550
2551             if ($scfg->{disable}) {
2552                 foreach my $node (keys %$nodehash) {
2553                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2554                 }
2555             } elsif (my $avail = $scfg->{nodes}) {
2556                 foreach my $node (keys %$nodehash) {
2557                     if (!$avail->{$node}) {
2558                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2559                     }
2560                 }
2561             }
2562         }
2563     });
2564
2565     foreach my $node (values %$nodehash) {
2566         if (my $unavail = $node->{unavailable_storages}) {
2567             $node->{unavailable_storages} = [ sort keys %$unavail ];
2568         }
2569     }
2570
2571     return $nodehash
2572 }
2573
2574 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2575 sub check_running {
2576     my ($vmid, $nocheck, $node) = @_;
2577
2578     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2579     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2580 }
2581
2582 sub vzlist {
2583
2584     my $vzlist = config_list();
2585
2586     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2587
2588     while (defined(my $de = $fd->read)) {
2589         next if $de !~ m/^(\d+)\.pid$/;
2590         my $vmid = $1;
2591         next if !defined($vzlist->{$vmid});
2592         if (my $pid = check_running($vmid)) {
2593             $vzlist->{$vmid}->{pid} = $pid;
2594         }
2595     }
2596
2597     return $vzlist;
2598 }
2599
2600 our $vmstatus_return_properties = {
2601     vmid => get_standard_option('pve-vmid'),
2602     status => {
2603         description => "Qemu process status.",
2604         type => 'string',
2605         enum => ['stopped', 'running'],
2606     },
2607     maxmem => {
2608         description => "Maximum memory in bytes.",
2609         type => 'integer',
2610         optional => 1,
2611         renderer => 'bytes',
2612     },
2613     maxdisk => {
2614         description => "Root disk size in bytes.",
2615         type => 'integer',
2616         optional => 1,
2617         renderer => 'bytes',
2618     },
2619     name => {
2620         description => "VM name.",
2621         type => 'string',
2622         optional => 1,
2623     },
2624     qmpstatus => {
2625         description => "Qemu QMP agent status.",
2626         type => 'string',
2627         optional => 1,
2628     },
2629     pid => {
2630         description => "PID of running qemu process.",
2631         type => 'integer',
2632         optional => 1,
2633     },
2634     uptime => {
2635         description => "Uptime.",
2636         type => 'integer',
2637         optional => 1,
2638         renderer => 'duration',
2639     },
2640     cpus => {
2641         description => "Maximum usable CPUs.",
2642         type => 'number',
2643         optional => 1,
2644     },
2645     lock => {
2646         description => "The current config lock, if any.",
2647         type => 'string',
2648         optional => 1,
2649     },
2650     tags => {
2651         description  => "The current configured tags, if any",
2652         type => 'string',
2653         optional => 1,
2654     },
2655     'running-machine' => {
2656         description => "The currently running machine type (if running).",
2657         type => 'string',
2658         optional => 1,
2659     },
2660     'running-qemu' => {
2661         description => "The currently running QEMU version (if running).",
2662         type => 'string',
2663         optional => 1,
2664     },
2665 };
2666
2667 my $last_proc_pid_stat;
2668
2669 # get VM status information
2670 # This must be fast and should not block ($full == false)
2671 # We only query KVM using QMP if $full == true (this can be slow)
2672 sub vmstatus {
2673     my ($opt_vmid, $full) = @_;
2674
2675     my $res = {};
2676
2677     my $storecfg = PVE::Storage::config();
2678
2679     my $list = vzlist();
2680     my $defaults = load_defaults();
2681
2682     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2683
2684     my $cpucount = $cpuinfo->{cpus} || 1;
2685
2686     foreach my $vmid (keys %$list) {
2687         next if $opt_vmid && ($vmid ne $opt_vmid);
2688
2689         my $conf = PVE::QemuConfig->load_config($vmid);
2690
2691         my $d = { vmid => int($vmid) };
2692         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2693
2694         # fixme: better status?
2695         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2696
2697         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2698         if (defined($size)) {
2699             $d->{disk} = 0; # no info available
2700             $d->{maxdisk} = $size;
2701         } else {
2702             $d->{disk} = 0;
2703             $d->{maxdisk} = 0;
2704         }
2705
2706         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2707             * ($conf->{cores} || $defaults->{cores});
2708         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2709         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2710
2711         $d->{name} = $conf->{name} || "VM $vmid";
2712         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2713             : $defaults->{memory}*(1024*1024);
2714
2715         if ($conf->{balloon}) {
2716             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2717             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2718                 : $defaults->{shares};
2719         }
2720
2721         $d->{uptime} = 0;
2722         $d->{cpu} = 0;
2723         $d->{mem} = 0;
2724
2725         $d->{netout} = 0;
2726         $d->{netin} = 0;
2727
2728         $d->{diskread} = 0;
2729         $d->{diskwrite} = 0;
2730
2731         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2732
2733         $d->{serial} = 1 if conf_has_serial($conf);
2734         $d->{lock} = $conf->{lock} if $conf->{lock};
2735         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2736
2737         $res->{$vmid} = $d;
2738     }
2739
2740     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2741     foreach my $dev (keys %$netdev) {
2742         next if $dev !~ m/^tap([1-9]\d*)i/;
2743         my $vmid = $1;
2744         my $d = $res->{$vmid};
2745         next if !$d;
2746
2747         $d->{netout} += $netdev->{$dev}->{receive};
2748         $d->{netin} += $netdev->{$dev}->{transmit};
2749
2750         if ($full) {
2751             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2752             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2753         }
2754
2755     }
2756
2757     my $ctime = gettimeofday;
2758
2759     foreach my $vmid (keys %$list) {
2760
2761         my $d = $res->{$vmid};
2762         my $pid = $d->{pid};
2763         next if !$pid;
2764
2765         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2766         next if !$pstat; # not running
2767
2768         my $used = $pstat->{utime} + $pstat->{stime};
2769
2770         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2771
2772         if ($pstat->{vsize}) {
2773             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2774         }
2775
2776         my $old = $last_proc_pid_stat->{$pid};
2777         if (!$old) {
2778             $last_proc_pid_stat->{$pid} = {
2779                 time => $ctime,
2780                 used => $used,
2781                 cpu => 0,
2782             };
2783             next;
2784         }
2785
2786         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2787
2788         if ($dtime > 1000) {
2789             my $dutime = $used -  $old->{used};
2790
2791             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2792             $last_proc_pid_stat->{$pid} = {
2793                 time => $ctime,
2794                 used => $used,
2795                 cpu => $d->{cpu},
2796             };
2797         } else {
2798             $d->{cpu} = $old->{cpu};
2799         }
2800     }
2801
2802     return $res if !$full;
2803
2804     my $qmpclient = PVE::QMPClient->new();
2805
2806     my $ballooncb = sub {
2807         my ($vmid, $resp) = @_;
2808
2809         my $info = $resp->{'return'};
2810         return if !$info->{max_mem};
2811
2812         my $d = $res->{$vmid};
2813
2814         # use memory assigned to VM
2815         $d->{maxmem} = $info->{max_mem};
2816         $d->{balloon} = $info->{actual};
2817
2818         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2819             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2820             $d->{freemem} = $info->{free_mem};
2821         }
2822
2823         $d->{ballooninfo} = $info;
2824     };
2825
2826     my $blockstatscb = sub {
2827         my ($vmid, $resp) = @_;
2828         my $data = $resp->{'return'} || [];
2829         my $totalrdbytes = 0;
2830         my $totalwrbytes = 0;
2831
2832         for my $blockstat (@$data) {
2833             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2834             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2835
2836             $blockstat->{device} =~ s/drive-//;
2837             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2838         }
2839         $res->{$vmid}->{diskread} = $totalrdbytes;
2840         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2841     };
2842
2843     my $machinecb = sub {
2844         my ($vmid, $resp) = @_;
2845         my $data = $resp->{'return'} || [];
2846
2847         $res->{$vmid}->{'running-machine'} =
2848             PVE::QemuServer::Machine::current_from_query_machines($data);
2849     };
2850
2851     my $versioncb = sub {
2852         my ($vmid, $resp) = @_;
2853         my $data = $resp->{'return'} // {};
2854         my $version = 'unknown';
2855
2856         if (my $v = $data->{qemu}) {
2857             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
2858         }
2859
2860         $res->{$vmid}->{'running-qemu'} = $version;
2861     };
2862
2863     my $statuscb = sub {
2864         my ($vmid, $resp) = @_;
2865
2866         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2867         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
2868         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
2869         # this fails if ballon driver is not loaded, so this must be
2870         # the last commnand (following command are aborted if this fails).
2871         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2872
2873         my $status = 'unknown';
2874         if (!defined($status = $resp->{'return'}->{status})) {
2875             warn "unable to get VM status\n";
2876             return;
2877         }
2878
2879         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2880     };
2881
2882     foreach my $vmid (keys %$list) {
2883         next if $opt_vmid && ($vmid ne $opt_vmid);
2884         next if !$res->{$vmid}->{pid}; # not running
2885         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2886     }
2887
2888     $qmpclient->queue_execute(undef, 2);
2889
2890     foreach my $vmid (keys %$list) {
2891         next if $opt_vmid && ($vmid ne $opt_vmid);
2892         next if !$res->{$vmid}->{pid}; #not running
2893
2894         # we can't use the $qmpclient since it might have already aborted on
2895         # 'query-balloon', but this might also fail for older versions...
2896         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
2897         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
2898     }
2899
2900     foreach my $vmid (keys %$list) {
2901         next if $opt_vmid && ($vmid ne $opt_vmid);
2902         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2903     }
2904
2905     return $res;
2906 }
2907
2908 sub conf_has_serial {
2909     my ($conf) = @_;
2910
2911     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2912         if ($conf->{"serial$i"}) {
2913             return 1;
2914         }
2915     }
2916
2917     return 0;
2918 }
2919
2920 sub conf_has_audio {
2921     my ($conf, $id) = @_;
2922
2923     $id //= 0;
2924     my $audio = $conf->{"audio$id"};
2925     return if !defined($audio);
2926
2927     my $audioproperties = parse_property_string($audio_fmt, $audio);
2928     my $audiodriver = $audioproperties->{driver} // 'spice';
2929
2930     return {
2931         dev => $audioproperties->{device},
2932         dev_id => "audiodev$id",
2933         backend => $audiodriver,
2934         backend_id => "$audiodriver-backend${id}",
2935     };
2936 }
2937
2938 sub audio_devs {
2939     my ($audio, $audiopciaddr, $machine_version) = @_;
2940
2941     my $devs = [];
2942
2943     my $id = $audio->{dev_id};
2944     my $audiodev = "";
2945     if (min_version($machine_version, 4, 2)) {
2946         $audiodev = ",audiodev=$audio->{backend_id}";
2947     }
2948
2949     if ($audio->{dev} eq 'AC97') {
2950         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
2951     } elsif ($audio->{dev} =~ /intel\-hda$/) {
2952         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
2953         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
2954         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
2955     } else {
2956         die "unkown audio device '$audio->{dev}', implement me!";
2957     }
2958
2959     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
2960
2961     return $devs;
2962 }
2963
2964 sub vga_conf_has_spice {
2965     my ($vga) = @_;
2966
2967     my $vgaconf = parse_vga($vga);
2968     my $vgatype = $vgaconf->{type};
2969     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
2970
2971     return $1 || 1;
2972 }
2973
2974 sub is_native($) {
2975     my ($arch) = @_;
2976     return get_host_arch() eq $arch;
2977 }
2978
2979 sub get_vm_arch {
2980     my ($conf) = @_;
2981     return $conf->{arch} // get_host_arch();
2982 }
2983
2984 my $default_machines = {
2985     x86_64 => 'pc',
2986     aarch64 => 'virt',
2987 };
2988
2989 sub get_installed_machine_version {
2990     my ($kvmversion) = @_;
2991     $kvmversion = kvm_user_version() if !defined($kvmversion);
2992     $kvmversion =~ m/^(\d+\.\d+)/;
2993     return $1;
2994 }
2995
2996 sub windows_get_pinned_machine_version {
2997     my ($machine, $base_version, $kvmversion) = @_;
2998
2999     my $pin_version = $base_version;
3000     if (!defined($base_version) ||
3001         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
3002     ) {
3003         $pin_version = get_installed_machine_version($kvmversion);
3004     }
3005     if (!$machine || $machine eq 'pc') {
3006         $machine = "pc-i440fx-$pin_version";
3007     } elsif ($machine eq 'q35') {
3008         $machine = "pc-q35-$pin_version";
3009     } elsif ($machine eq 'virt') {
3010         $machine = "virt-$pin_version";
3011     } else {
3012         warn "unknown machine type '$machine', not touching that!\n";
3013     }
3014
3015     return $machine;
3016 }
3017
3018 sub get_vm_machine {
3019     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3020
3021     my $machine = $forcemachine || $conf->{machine};
3022
3023     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3024         $kvmversion //= kvm_user_version();
3025         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3026         # layout which confuses windows quite a bit and may result in various regressions..
3027         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3028         if (windows_version($conf->{ostype})) {
3029             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3030         }
3031         $arch //= 'x86_64';
3032         $machine ||= $default_machines->{$arch};
3033         if ($add_pve_version) {
3034             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3035             $machine .= "+pve$pvever";
3036         }
3037     }
3038
3039     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3040         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3041         $machine = $1 if $is_pxe;
3042
3043         # for version-pinned machines that do not include a pve-version (e.g.
3044         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3045         $machine .= '+pve0';
3046
3047         $machine .= '.pxe' if $is_pxe;
3048     }
3049
3050     return $machine;
3051 }
3052
3053 sub get_ovmf_files($) {
3054     my ($arch) = @_;
3055
3056     my $ovmf = $OVMF->{$arch}
3057         or die "no OVMF images known for architecture '$arch'\n";
3058
3059     return @$ovmf;
3060 }
3061
3062 my $Arch2Qemu = {
3063     aarch64 => '/usr/bin/qemu-system-aarch64',
3064     x86_64 => '/usr/bin/qemu-system-x86_64',
3065 };
3066 sub get_command_for_arch($) {
3067     my ($arch) = @_;
3068     return '/usr/bin/kvm' if is_native($arch);
3069
3070     my $cmd = $Arch2Qemu->{$arch}
3071         or die "don't know how to emulate architecture '$arch'\n";
3072     return $cmd;
3073 }
3074
3075 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3076 # to use in a QEMU command line (-cpu element), first array_intersect the result
3077 # of query_supported_ with query_understood_. This is necessary because:
3078 #
3079 # a) query_understood_ returns flags the host cannot use and
3080 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3081 #    flags, but CPU settings - with most of them being flags. Those settings
3082 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3083 #
3084 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3085 # expensive. If you need the value returned from this, you can get it much
3086 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3087 # $accel being 'kvm' or 'tcg'.
3088 #
3089 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3090 # changes, automatically populating pmxcfs.
3091 #
3092 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3093 # since kvm and tcg machines support different flags
3094 #
3095 sub query_supported_cpu_flags {
3096     my ($arch) = @_;
3097
3098     $arch //= get_host_arch();
3099     my $default_machine = $default_machines->{$arch};
3100
3101     my $flags = {};
3102
3103     # FIXME: Once this is merged, the code below should work for ARM as well:
3104     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3105     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3106         $arch eq "aarch64";
3107
3108     my $kvm_supported = defined(kvm_version());
3109     my $qemu_cmd = get_command_for_arch($arch);
3110     my $fakevmid = -1;
3111     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3112
3113     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3114     my $query_supported_run_qemu = sub {
3115         my ($kvm) = @_;
3116
3117         my $flags = {};
3118         my $cmd = [
3119             $qemu_cmd,
3120             '-machine', $default_machine,
3121             '-display', 'none',
3122             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3123             '-mon', 'chardev=qmp,mode=control',
3124             '-pidfile', $pidfile,
3125             '-S', '-daemonize'
3126         ];
3127
3128         if (!$kvm) {
3129             push @$cmd, '-accel', 'tcg';
3130         }
3131
3132         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3133         die "QEMU flag querying VM exited with code " . $rc if $rc;
3134
3135         eval {
3136             my $cmd_result = mon_cmd(
3137                 $fakevmid,
3138                 'query-cpu-model-expansion',
3139                 type => 'full',
3140                 model => { name => 'host' }
3141             );
3142
3143             my $props = $cmd_result->{model}->{props};
3144             foreach my $prop (keys %$props) {
3145                 next if $props->{$prop} ne '1';
3146                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3147                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3148                 # We only keep those with underscores, to match /proc/cpuinfo
3149                 $prop =~ s/\.|-/_/g;
3150                 $flags->{$prop} = 1;
3151             }
3152         };
3153         my $err = $@;
3154
3155         # force stop with 10 sec timeout and 'nocheck'
3156         # always stop, even if QMP failed
3157         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3158
3159         die $err if $err;
3160
3161         return [ sort keys %$flags ];
3162     };
3163
3164     # We need to query QEMU twice, since KVM and TCG have different supported flags
3165     PVE::QemuConfig->lock_config($fakevmid, sub {
3166         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3167         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3168
3169         if ($kvm_supported) {
3170             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3171             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3172         }
3173     });
3174
3175     return $flags;
3176 }
3177
3178 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3179 my $understood_cpu_flag_dir = "/usr/share/kvm";
3180 sub query_understood_cpu_flags {
3181     my $arch = get_host_arch();
3182     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3183
3184     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3185         if ! -e $filepath;
3186
3187     my $raw = file_get_contents($filepath);
3188     $raw =~ s/^\s+|\s+$//g;
3189     my @flags = split(/\s+/, $raw);
3190
3191     return \@flags;
3192 }
3193
3194 my sub get_cpuunits {
3195     my ($conf) = @_;
3196     return $conf->{cpuunits} // (PVE::CGroup::cgroup_mode() == 2 ? 100 : 1024);
3197 }
3198 sub config_to_command {
3199     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3200         $pbs_backing) = @_;
3201
3202     my $cmd = [];
3203     my $globalFlags = [];
3204     my $machineFlags = [];
3205     my $rtcFlags = [];
3206     my $devices = [];
3207     my $pciaddr = '';
3208     my $bridges = {};
3209     my $ostype = $conf->{ostype};
3210     my $winversion = windows_version($ostype);
3211     my $kvm = $conf->{kvm};
3212     my $nodename = nodename();
3213
3214     my $arch = get_vm_arch($conf);
3215     my $kvm_binary = get_command_for_arch($arch);
3216     my $kvmver = kvm_user_version($kvm_binary);
3217
3218     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3219         $kvmver //= "undefined";
3220         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3221     }
3222
3223     my $add_pve_version = min_version($kvmver, 4, 1);
3224
3225     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3226     my $machine_version = extract_version($machine_type, $kvmver);
3227     $kvm //= 1 if is_native($arch);
3228
3229     $machine_version =~ m/(\d+)\.(\d+)/;
3230     my ($machine_major, $machine_minor) = ($1, $2);
3231
3232     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3233         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3234     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3235         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3236             ." please upgrade node '$nodename'\n"
3237     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3238         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3239         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3240             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3241             ." node '$nodename'\n";
3242     }
3243
3244     # if a specific +pve version is required for a feature, use $version_guard
3245     # instead of min_version to allow machines to be run with the minimum
3246     # required version
3247     my $required_pve_version = 0;
3248     my $version_guard = sub {
3249         my ($major, $minor, $pve) = @_;
3250         return 0 if !min_version($machine_version, $major, $minor, $pve);
3251         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3252         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3253         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3254         return 1;
3255     };
3256
3257     if ($kvm && !defined kvm_version()) {
3258         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3259             ." or enable in BIOS.\n";
3260     }
3261
3262     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3263     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3264     my $use_old_bios_files = undef;
3265     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3266
3267     my $cpuunits = get_cpuunits($conf);
3268
3269     push @$cmd, $kvm_binary;
3270
3271     push @$cmd, '-id', $vmid;
3272
3273     my $vmname = $conf->{name} || "vm$vmid";
3274
3275     push @$cmd, '-name', $vmname;
3276
3277     push @$cmd, '-no-shutdown';
3278
3279     my $use_virtio = 0;
3280
3281     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3282     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3283     push @$cmd, '-mon', "chardev=qmp,mode=control";
3284
3285     if (min_version($machine_version, 2, 12)) {
3286         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3287         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3288     }
3289
3290     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3291
3292     push @$cmd, '-daemonize';
3293
3294     if ($conf->{smbios1}) {
3295         my $smbios_conf = parse_smbios1($conf->{smbios1});
3296         if ($smbios_conf->{base64}) {
3297             # Do not pass base64 flag to qemu
3298             delete $smbios_conf->{base64};
3299             my $smbios_string = "";
3300             foreach my $key (keys %$smbios_conf) {
3301                 my $value;
3302                 if ($key eq "uuid") {
3303                     $value = $smbios_conf->{uuid}
3304                 } else {
3305                     $value = decode_base64($smbios_conf->{$key});
3306                 }
3307                 # qemu accepts any binary data, only commas need escaping by double comma
3308                 $value =~ s/,/,,/g;
3309                 $smbios_string .= "," . $key . "=" . $value if $value;
3310             }
3311             push @$cmd, '-smbios', "type=1" . $smbios_string;
3312         } else {
3313             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3314         }
3315     }
3316
3317     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3318         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3319         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3320
3321         my ($path, $format);
3322         my $read_only_str = '';
3323         if (my $efidisk = $conf->{efidisk0}) {
3324             my $d = parse_drive('efidisk0', $efidisk);
3325             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3326             $format = $d->{format};
3327             if ($storeid) {
3328                 $path = PVE::Storage::path($storecfg, $d->{file});
3329                 if (!defined($format)) {
3330                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3331                     $format = qemu_img_format($scfg, $volname);
3332                 }
3333             } else {
3334                 $path = $d->{file};
3335                 die "efidisk format must be specified\n"
3336                     if !defined($format);
3337             }
3338
3339             $read_only_str = ',readonly=on' if drive_is_read_only($conf, $d);
3340         } else {
3341             warn "no efidisk configured! Using temporary efivars disk.\n";
3342             $path = "/tmp/$vmid-ovmf.fd";
3343             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3344             $format = 'raw';
3345         }
3346
3347         my $size_str = "";
3348
3349         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3350             $size_str = ",size=" . (-s $ovmf_vars);
3351         }
3352
3353         # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3354         my $cache = "";
3355         if ($path =~ m/^rbd:/) {
3356                 $cache = ',cache=writeback';
3357                 $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3358         }
3359
3360         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code";
3361         push @$cmd, '-drive', "if=pflash,unit=1$cache,format=$format,id=drive-efidisk0$size_str,file=${path}${read_only_str}";
3362     }
3363
3364     # load q35 config
3365     if ($q35) {
3366         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3367         if (min_version($machine_version, 4, 0)) {
3368             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3369         } else {
3370             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3371         }
3372     }
3373
3374     if ($conf->{vmgenid}) {
3375         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3376     }
3377
3378     # add usb controllers
3379     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3380         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3381     push @$devices, @usbcontrollers if @usbcontrollers;
3382     my $vga = parse_vga($conf->{vga});
3383
3384     my $qxlnum = vga_conf_has_spice($conf->{vga});
3385     $vga->{type} = 'qxl' if $qxlnum;
3386
3387     if (!$vga->{type}) {
3388         if ($arch eq 'aarch64') {
3389             $vga->{type} = 'virtio';
3390         } elsif (min_version($machine_version, 2, 9)) {
3391             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3392         } else {
3393             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3394         }
3395     }
3396
3397     # enable absolute mouse coordinates (needed by vnc)
3398     my $tablet;
3399     if (defined($conf->{tablet})) {
3400         $tablet = $conf->{tablet};
3401     } else {
3402         $tablet = $defaults->{tablet};
3403         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3404         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3405     }
3406
3407     if ($tablet) {
3408         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3409         my $kbd = print_keyboarddevice_full($conf, $arch);
3410         push @$devices, '-device', $kbd if defined($kbd);
3411     }
3412
3413     my $bootorder = device_bootorder($conf);
3414
3415     # host pci device passthrough
3416     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3417         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3418
3419     # usb devices
3420     my $usb_dev_features = {};
3421     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3422
3423     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3424         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3425     push @$devices, @usbdevices if @usbdevices;
3426
3427     # serial devices
3428     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3429         if (my $path = $conf->{"serial$i"}) {
3430             if ($path eq 'socket') {
3431                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3432                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3433                 # On aarch64, serial0 is the UART device. Qemu only allows
3434                 # connecting UART devices via the '-serial' command line, as
3435                 # the device has a fixed slot on the hardware...
3436                 if ($arch eq 'aarch64' && $i == 0) {
3437                     push @$devices, '-serial', "chardev:serial$i";
3438                 } else {
3439                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3440                 }
3441             } else {
3442                 die "no such serial device\n" if ! -c $path;
3443                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3444                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3445             }
3446         }
3447     }
3448
3449     # parallel devices
3450     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3451         if (my $path = $conf->{"parallel$i"}) {
3452             die "no such parallel device\n" if ! -c $path;
3453             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3454             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3455             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3456         }
3457     }
3458
3459     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3460         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3461         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3462         push @$devices, @$audio_devs;
3463     }
3464
3465     my $sockets = 1;
3466     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3467     $sockets = $conf->{sockets} if  $conf->{sockets};
3468
3469     my $cores = $conf->{cores} || 1;
3470
3471     my $maxcpus = $sockets * $cores;
3472
3473     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3474
3475     my $allowed_vcpus = $cpuinfo->{cpus};
3476
3477     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3478         if ($allowed_vcpus < $maxcpus);
3479
3480     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3481
3482         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3483         for (my $i = 2; $i <= $vcpus; $i++)  {
3484             my $cpustr = print_cpu_device($conf,$i);
3485             push @$cmd, '-device', $cpustr;
3486         }
3487
3488     } else {
3489
3490         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3491     }
3492     push @$cmd, '-nodefaults';
3493
3494     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3495
3496     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3497
3498     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3499
3500     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3501         push @$devices, '-device', print_vga_device(
3502             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3503         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3504         push @$cmd,  '-vnc', "unix:$socket,password=on";
3505     } else {
3506         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3507         push @$cmd, '-nographic';
3508     }
3509
3510     # time drift fix
3511     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3512     my $useLocaltime = $conf->{localtime};
3513
3514     if ($winversion >= 5) { # windows
3515         $useLocaltime = 1 if !defined($conf->{localtime});
3516
3517         # use time drift fix when acpi is enabled
3518         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3519             $tdf = 1 if !defined($conf->{tdf});
3520         }
3521     }
3522
3523     if ($winversion >= 6) {
3524         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3525         push @$cmd, '-no-hpet';
3526     }
3527
3528     push @$rtcFlags, 'driftfix=slew' if $tdf;
3529
3530     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3531         push @$rtcFlags, "base=$conf->{startdate}";
3532     } elsif ($useLocaltime) {
3533         push @$rtcFlags, 'base=localtime';
3534     }
3535
3536     if ($forcecpu) {
3537         push @$cmd, '-cpu', $forcecpu;
3538     } else {
3539         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3540     }
3541
3542     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3543
3544     push @$cmd, '-S' if $conf->{freeze};
3545
3546     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3547
3548     my $guest_agent = parse_guest_agent($conf);
3549
3550     if ($guest_agent->{enabled}) {
3551         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3552         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3553
3554         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3555             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3556             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3557             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3558         } elsif ($guest_agent->{type} eq 'isa') {
3559             push @$devices, '-device', "isa-serial,chardev=qga0";
3560         }
3561     }
3562
3563     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3564     if ($rng && $version_guard->(4, 1, 2)) {
3565         check_rng_source($rng->{source});
3566
3567         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3568         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3569         my $limiter_str = "";
3570         if ($max_bytes) {
3571             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3572         }
3573
3574         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3575         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3576         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3577     }
3578
3579     my $spice_port;
3580
3581     if ($qxlnum) {
3582         if ($qxlnum > 1) {
3583             if ($winversion){
3584                 for (my $i = 1; $i < $qxlnum; $i++){
3585                     push @$devices, '-device', print_vga_device(
3586                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3587                 }
3588             } else {
3589                 # assume other OS works like Linux
3590                 my ($ram, $vram) = ("134217728", "67108864");
3591                 if ($vga->{memory}) {
3592                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3593                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3594                 }
3595                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3596                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3597             }
3598         }
3599
3600         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3601
3602         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3603         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3604         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3605
3606         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3607         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3608         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3609
3610         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3611         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3612
3613         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3614         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3615         if ($spice_enhancement->{foldersharing}) {
3616             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3617             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3618         }
3619
3620         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3621         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3622             if $spice_enhancement->{videostreaming};
3623
3624         push @$devices, '-spice', "$spice_opts";
3625     }
3626
3627     # enable balloon by default, unless explicitly disabled
3628     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3629         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3630         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3631     }
3632
3633     if ($conf->{watchdog}) {
3634         my $wdopts = parse_watchdog($conf->{watchdog});
3635         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3636         my $watchdog = $wdopts->{model} || 'i6300esb';
3637         push @$devices, '-device', "$watchdog$pciaddr";
3638         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3639     }
3640
3641     my $vollist = [];
3642     my $scsicontroller = {};
3643     my $ahcicontroller = {};
3644     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3645
3646     # Add iscsi initiator name if available
3647     if (my $initiator = get_initiator_name()) {
3648         push @$devices, '-iscsi', "initiator-name=$initiator";
3649     }
3650
3651     PVE::QemuConfig->foreach_volume($conf, sub {
3652         my ($ds, $drive) = @_;
3653
3654         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3655             check_volume_storage_type($storecfg, $drive->{file});
3656             push @$vollist, $drive->{file};
3657         }
3658
3659         # ignore efidisk here, already added in bios/fw handling code above
3660         return if $drive->{interface} eq 'efidisk';
3661
3662         $use_virtio = 1 if $ds =~ m/^virtio/;
3663
3664         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3665
3666         if ($drive->{interface} eq 'virtio'){
3667            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3668         }
3669
3670         if ($drive->{interface} eq 'scsi') {
3671
3672             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3673
3674             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3675                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3676
3677             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3678             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3679
3680             my $iothread = '';
3681             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3682                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3683                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3684             } elsif ($drive->{iothread}) {
3685                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3686             }
3687
3688             my $queues = '';
3689             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3690                 $queues = ",num_queues=$drive->{queues}";
3691             }
3692
3693             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3694                 if !$scsicontroller->{$controller};
3695             $scsicontroller->{$controller}=1;
3696         }
3697
3698         if ($drive->{interface} eq 'sata') {
3699             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3700             $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3701             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3702                 if !$ahcicontroller->{$controller};
3703             $ahcicontroller->{$controller}=1;
3704         }
3705
3706         my $pbs_conf = $pbs_backing->{$ds};
3707         my $pbs_name = undef;
3708         if ($pbs_conf) {
3709             $pbs_name = "drive-$ds-pbs";
3710             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
3711         }
3712
3713         my $drive_cmd = print_drive_commandline_full(
3714             $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
3715
3716         # extra protection for templates, but SATA and IDE don't support it..
3717         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
3718
3719         push @$devices, '-drive',$drive_cmd;
3720         push @$devices, '-device', print_drivedevice_full(
3721             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3722     });
3723
3724     for (my $i = 0; $i < $MAX_NETS; $i++) {
3725         my $netname = "net$i";
3726
3727         next if !$conf->{$netname};
3728         my $d = parse_net($conf->{$netname});
3729         next if !$d;
3730
3731         $use_virtio = 1 if $d->{model} eq 'virtio';
3732
3733         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
3734
3735         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
3736         push @$devices, '-netdev', $netdevfull;
3737
3738         my $netdevicefull = print_netdevice_full(
3739             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
3740
3741         push @$devices, '-device', $netdevicefull;
3742     }
3743
3744     if ($conf->{ivshmem}) {
3745         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3746
3747         my $bus;
3748         if ($q35) {
3749             $bus = print_pcie_addr("ivshmem");
3750         } else {
3751             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3752         }
3753
3754         my $ivshmem_name = $ivshmem->{name} // $vmid;
3755         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3756
3757         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3758         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
3759             .",size=$ivshmem->{size}M";
3760     }
3761
3762     # pci.4 is nested in pci.1
3763     $bridges->{1} = 1 if $bridges->{4};
3764
3765     if (!$q35) {
3766         # add pci bridges
3767         if (min_version($machine_version, 2, 3)) {
3768            $bridges->{1} = 1;
3769            $bridges->{2} = 1;
3770         }
3771
3772         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3773
3774     }
3775
3776     for my $k (sort {$b cmp $a} keys %$bridges) {
3777         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3778
3779         my $k_name = $k;
3780         if ($k == 2 && $legacy_igd) {
3781             $k_name = "$k-igd";
3782         }
3783         $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
3784
3785         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3786         if ($q35) {
3787             # add after -readconfig pve-q35.cfg
3788             splice @$devices, 2, 0, '-device', $devstr;
3789         } else {
3790             unshift @$devices, '-device', $devstr if $k > 0;
3791         }
3792     }
3793
3794     if (!$kvm) {
3795         push @$machineFlags, 'accel=tcg';
3796     }
3797
3798     my $machine_type_min = $machine_type;
3799     if ($add_pve_version) {
3800         $machine_type_min =~ s/\+pve\d+$//;
3801         $machine_type_min .= "+pve$required_pve_version";
3802     }
3803     push @$machineFlags, "type=${machine_type_min}";
3804
3805     push @$cmd, @$devices;
3806     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
3807     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
3808     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
3809
3810     if (my $vmstate = $conf->{vmstate}) {
3811         my $statepath = PVE::Storage::path($storecfg, $vmstate);
3812         push @$vollist, $vmstate;
3813         push @$cmd, '-loadstate', $statepath;
3814         print "activating and using '$vmstate' as vmstate\n";
3815     }
3816
3817     if (PVE::QemuConfig->is_template($conf)) {
3818         # needed to workaround base volumes being read-only
3819         push @$cmd, '-snapshot';
3820     }
3821
3822     # add custom args
3823     if ($conf->{args}) {
3824         my $aa = PVE::Tools::split_args($conf->{args});
3825         push @$cmd, @$aa;
3826     }
3827
3828     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3829 }
3830
3831 sub check_rng_source {
3832     my ($source) = @_;
3833
3834     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3835     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
3836         if ! -e $source;
3837
3838     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
3839     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
3840         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
3841         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
3842         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
3843             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
3844             ." to the host.\n";
3845     }
3846 }
3847
3848 sub spice_port {
3849     my ($vmid) = @_;
3850
3851     my $res = mon_cmd($vmid, 'query-spice');
3852
3853     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3854 }
3855
3856 sub vm_devices_list {
3857     my ($vmid) = @_;
3858
3859     my $res = mon_cmd($vmid, 'query-pci');
3860     my $devices_to_check = [];
3861     my $devices = {};
3862     foreach my $pcibus (@$res) {
3863         push @$devices_to_check, @{$pcibus->{devices}},
3864     }
3865
3866     while (@$devices_to_check) {
3867         my $to_check = [];
3868         for my $d (@$devices_to_check) {
3869             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3870             next if !$d->{'pci_bridge'};
3871
3872             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3873             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3874         }
3875         $devices_to_check = $to_check;
3876     }
3877
3878     my $resblock = mon_cmd($vmid, 'query-block');
3879     foreach my $block (@$resblock) {
3880         if($block->{device} =~ m/^drive-(\S+)/){
3881                 $devices->{$1} = 1;
3882         }
3883     }
3884
3885     my $resmice = mon_cmd($vmid, 'query-mice');
3886     foreach my $mice (@$resmice) {
3887         if ($mice->{name} eq 'QEMU HID Tablet') {
3888             $devices->{tablet} = 1;
3889             last;
3890         }
3891     }
3892
3893     # for usb devices there is no query-usb
3894     # but we can iterate over the entries in
3895     # qom-list path=/machine/peripheral
3896     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3897     foreach my $per (@$resperipheral) {
3898         if ($per->{name} =~ m/^usb\d+$/) {
3899             $devices->{$per->{name}} = 1;
3900         }
3901     }
3902
3903     return $devices;
3904 }
3905
3906 sub vm_deviceplug {
3907     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3908
3909     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3910
3911     my $devices_list = vm_devices_list($vmid);
3912     return 1 if defined($devices_list->{$deviceid});
3913
3914     # add PCI bridge if we need it for the device
3915     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
3916
3917     if ($deviceid eq 'tablet') {
3918
3919         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
3920
3921     } elsif ($deviceid eq 'keyboard') {
3922
3923         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
3924
3925     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3926
3927         die "usb hotplug currently not reliable\n";
3928         # since we can't reliably hot unplug all added usb devices and usb
3929         # passthrough breaks live migration we disable usb hotplugging for now
3930         #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3931
3932     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3933
3934         qemu_iothread_add($vmid, $deviceid, $device);
3935
3936         qemu_driveadd($storecfg, $vmid, $device);
3937         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3938
3939         qemu_deviceadd($vmid, $devicefull);
3940         eval { qemu_deviceaddverify($vmid, $deviceid); };
3941         if (my $err = $@) {
3942             eval { qemu_drivedel($vmid, $deviceid); };
3943             warn $@ if $@;
3944             die $err;
3945         }
3946
3947     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3948
3949
3950         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3951         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3952         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3953
3954         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3955
3956         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3957             qemu_iothread_add($vmid, $deviceid, $device);
3958             $devicefull .= ",iothread=iothread-$deviceid";
3959         }
3960
3961         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3962             $devicefull .= ",num_queues=$device->{queues}";
3963         }
3964
3965         qemu_deviceadd($vmid, $devicefull);
3966         qemu_deviceaddverify($vmid, $deviceid);
3967
3968     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3969
3970         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
3971         qemu_driveadd($storecfg, $vmid, $device);
3972
3973         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3974         eval { qemu_deviceadd($vmid, $devicefull); };
3975         if (my $err = $@) {
3976             eval { qemu_drivedel($vmid, $deviceid); };
3977             warn $@ if $@;
3978             die $err;
3979         }
3980
3981     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3982
3983         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
3984
3985         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
3986         my $use_old_bios_files = undef;
3987         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3988
3989         my $netdevicefull = print_netdevice_full(
3990             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
3991         qemu_deviceadd($vmid, $netdevicefull);
3992         eval {
3993             qemu_deviceaddverify($vmid, $deviceid);
3994             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
3995         };
3996         if (my $err = $@) {
3997             eval { qemu_netdevdel($vmid, $deviceid); };
3998             warn $@ if $@;
3999             die $err;
4000         }
4001
4002     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4003
4004         my $bridgeid = $2;
4005         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4006         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4007
4008         qemu_deviceadd($vmid, $devicefull);
4009         qemu_deviceaddverify($vmid, $deviceid);
4010
4011     } else {
4012         die "can't hotplug device '$deviceid'\n";
4013     }
4014
4015     return 1;
4016 }
4017
4018 # fixme: this should raise exceptions on error!
4019 sub vm_deviceunplug {
4020     my ($vmid, $conf, $deviceid) = @_;
4021
4022     my $devices_list = vm_devices_list($vmid);
4023     return 1 if !defined($devices_list->{$deviceid});
4024
4025     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
4026     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4027
4028     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4029         qemu_devicedel($vmid, $deviceid);
4030     } elsif ($deviceid =~ m/^usb\d+$/) {
4031         die "usb hotplug currently not reliable\n";
4032         # when unplugging usb devices this way, there may be remaining usb
4033         # controllers/hubs so we disable it for now
4034         #qemu_devicedel($vmid, $deviceid);
4035         #qemu_devicedelverify($vmid, $deviceid);
4036     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4037         my $device = parse_drive($deviceid, $conf->{$deviceid});
4038
4039         qemu_devicedel($vmid, $deviceid);
4040         qemu_devicedelverify($vmid, $deviceid);
4041         qemu_drivedel($vmid, $deviceid);
4042         qemu_iothread_del($vmid, $deviceid, $device);
4043     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4044         qemu_devicedel($vmid, $deviceid);
4045         qemu_devicedelverify($vmid, $deviceid);
4046     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4047         my $device = parse_drive($deviceid, $conf->{$deviceid});
4048
4049         qemu_devicedel($vmid, $deviceid);
4050         qemu_drivedel($vmid, $deviceid);
4051         qemu_deletescsihw($conf, $vmid, $deviceid);
4052
4053         qemu_iothread_del($vmid, "virtioscsi$device->{index}", $device)
4054             if $conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single');
4055     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4056         qemu_devicedel($vmid, $deviceid);
4057         qemu_devicedelverify($vmid, $deviceid);
4058         qemu_netdevdel($vmid, $deviceid);
4059     } else {
4060         die "can't unplug device '$deviceid'\n";
4061     }
4062
4063     return 1;
4064 }
4065
4066 sub qemu_deviceadd {
4067     my ($vmid, $devicefull) = @_;
4068
4069     $devicefull = "driver=".$devicefull;
4070     my %options =  split(/[=,]/, $devicefull);
4071
4072     mon_cmd($vmid, "device_add" , %options);
4073 }
4074
4075 sub qemu_devicedel {
4076     my ($vmid, $deviceid) = @_;
4077
4078     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4079 }
4080
4081 sub qemu_iothread_add {
4082     my ($vmid, $deviceid, $device) = @_;
4083
4084     if ($device->{iothread}) {
4085         my $iothreads = vm_iothreads_list($vmid);
4086         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4087     }
4088 }
4089
4090 sub qemu_iothread_del {
4091     my ($vmid, $deviceid, $device) = @_;
4092
4093     if ($device->{iothread}) {
4094         my $iothreads = vm_iothreads_list($vmid);
4095         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4096     }
4097 }
4098
4099 sub qemu_objectadd {
4100     my ($vmid, $objectid, $qomtype) = @_;
4101
4102     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4103
4104     return 1;
4105 }
4106
4107 sub qemu_objectdel {
4108     my ($vmid, $objectid) = @_;
4109
4110     mon_cmd($vmid, "object-del", id => $objectid);
4111
4112     return 1;
4113 }
4114
4115 sub qemu_driveadd {
4116     my ($storecfg, $vmid, $device) = @_;
4117
4118     my $kvmver = get_running_qemu_version($vmid);
4119     my $io_uring = min_version($kvmver, 6, 0);
4120     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4121     $drive =~ s/\\/\\\\/g;
4122     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4123
4124     # If the command succeeds qemu prints: "OK"
4125     return 1 if $ret =~ m/OK/s;
4126
4127     die "adding drive failed: $ret\n";
4128 }
4129
4130 sub qemu_drivedel {
4131     my ($vmid, $deviceid) = @_;
4132
4133     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4134     $ret =~ s/^\s+//;
4135
4136     return 1 if $ret eq "";
4137
4138     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4139     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4140
4141     die "deleting drive $deviceid failed : $ret\n";
4142 }
4143
4144 sub qemu_deviceaddverify {
4145     my ($vmid, $deviceid) = @_;
4146
4147     for (my $i = 0; $i <= 5; $i++) {
4148          my $devices_list = vm_devices_list($vmid);
4149          return 1 if defined($devices_list->{$deviceid});
4150          sleep 1;
4151     }
4152
4153     die "error on hotplug device '$deviceid'\n";
4154 }
4155
4156
4157 sub qemu_devicedelverify {
4158     my ($vmid, $deviceid) = @_;
4159
4160     # need to verify that the device is correctly removed as device_del
4161     # is async and empty return is not reliable
4162
4163     for (my $i = 0; $i <= 5; $i++) {
4164          my $devices_list = vm_devices_list($vmid);
4165          return 1 if !defined($devices_list->{$deviceid});
4166          sleep 1;
4167     }
4168
4169     die "error on hot-unplugging device '$deviceid'\n";
4170 }
4171
4172 sub qemu_findorcreatescsihw {
4173     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4174
4175     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4176
4177     my $scsihwid="$controller_prefix$controller";
4178     my $devices_list = vm_devices_list($vmid);
4179
4180     if (!defined($devices_list->{$scsihwid})) {
4181         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4182     }
4183
4184     return 1;
4185 }
4186
4187 sub qemu_deletescsihw {
4188     my ($conf, $vmid, $opt) = @_;
4189
4190     my $device = parse_drive($opt, $conf->{$opt});
4191
4192     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4193         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4194         return 1;
4195     }
4196
4197     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4198
4199     my $devices_list = vm_devices_list($vmid);
4200     foreach my $opt (keys %{$devices_list}) {
4201         if (is_valid_drivename($opt)) {
4202             my $drive = parse_drive($opt, $conf->{$opt});
4203             if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4204                 return 1;
4205             }
4206         }
4207     }
4208
4209     my $scsihwid="scsihw$controller";
4210
4211     vm_deviceunplug($vmid, $conf, $scsihwid);
4212
4213     return 1;
4214 }
4215
4216 sub qemu_add_pci_bridge {
4217     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4218
4219     my $bridges = {};
4220
4221     my $bridgeid;
4222
4223     print_pci_addr($device, $bridges, $arch, $machine_type);
4224
4225     while (my ($k, $v) = each %$bridges) {
4226         $bridgeid = $k;
4227     }
4228     return 1 if !defined($bridgeid) || $bridgeid < 1;
4229
4230     my $bridge = "pci.$bridgeid";
4231     my $devices_list = vm_devices_list($vmid);
4232
4233     if (!defined($devices_list->{$bridge})) {
4234         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4235     }
4236
4237     return 1;
4238 }
4239
4240 sub qemu_set_link_status {
4241     my ($vmid, $device, $up) = @_;
4242
4243     mon_cmd($vmid, "set_link", name => $device,
4244                up => $up ? JSON::true : JSON::false);
4245 }
4246
4247 sub qemu_netdevadd {
4248     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4249
4250     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4251     my %options =  split(/[=,]/, $netdev);
4252
4253     if (defined(my $vhost = $options{vhost})) {
4254         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4255     }
4256
4257     if (defined(my $queues = $options{queues})) {
4258         $options{queues} = $queues + 0;
4259     }
4260
4261     mon_cmd($vmid, "netdev_add",  %options);
4262     return 1;
4263 }
4264
4265 sub qemu_netdevdel {
4266     my ($vmid, $deviceid) = @_;
4267
4268     mon_cmd($vmid, "netdev_del", id => $deviceid);
4269 }
4270
4271 sub qemu_usb_hotplug {
4272     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4273
4274     return if !$device;
4275
4276     # remove the old one first
4277     vm_deviceunplug($vmid, $conf, $deviceid);
4278
4279     # check if xhci controller is necessary and available
4280     if ($device->{usb3}) {
4281
4282         my $devicelist = vm_devices_list($vmid);
4283
4284         if (!$devicelist->{xhci}) {
4285             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4286             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4287         }
4288     }
4289     my $d = parse_usb_device($device->{host});
4290     $d->{usb3} = $device->{usb3};
4291
4292     # add the new one
4293     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4294 }
4295
4296 sub qemu_cpu_hotplug {
4297     my ($vmid, $conf, $vcpus) = @_;
4298
4299     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4300
4301     my $sockets = 1;
4302     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4303     $sockets = $conf->{sockets} if  $conf->{sockets};
4304     my $cores = $conf->{cores} || 1;
4305     my $maxcpus = $sockets * $cores;
4306
4307     $vcpus = $maxcpus if !$vcpus;
4308
4309     die "you can't add more vcpus than maxcpus\n"
4310         if $vcpus > $maxcpus;
4311
4312     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4313
4314     if ($vcpus < $currentvcpus) {
4315
4316         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4317
4318             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4319                 qemu_devicedel($vmid, "cpu$i");
4320                 my $retry = 0;
4321                 my $currentrunningvcpus = undef;
4322                 while (1) {
4323                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4324                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4325                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4326                     $retry++;
4327                     sleep 1;
4328                 }
4329                 #update conf after each succesfull cpu unplug
4330                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4331                 PVE::QemuConfig->write_config($vmid, $conf);
4332             }
4333         } else {
4334             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4335         }
4336
4337         return;
4338     }
4339
4340     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4341     die "vcpus in running vm does not match its configuration\n"
4342         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4343
4344     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4345
4346         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4347             my $cpustr = print_cpu_device($conf, $i);
4348             qemu_deviceadd($vmid, $cpustr);
4349
4350             my $retry = 0;
4351             my $currentrunningvcpus = undef;
4352             while (1) {
4353                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4354                 last if scalar(@{$currentrunningvcpus}) == $i;
4355                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4356                 sleep 1;
4357                 $retry++;
4358             }
4359             #update conf after each succesfull cpu hotplug
4360             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4361             PVE::QemuConfig->write_config($vmid, $conf);
4362         }
4363     } else {
4364
4365         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4366             mon_cmd($vmid, "cpu-add", id => int($i));
4367         }
4368     }
4369 }
4370
4371 sub qemu_block_set_io_throttle {
4372     my ($vmid, $deviceid,
4373         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4374         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4375         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4376         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4377
4378     return if !check_running($vmid) ;
4379
4380     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4381         bps => int($bps),
4382         bps_rd => int($bps_rd),
4383         bps_wr => int($bps_wr),
4384         iops => int($iops),
4385         iops_rd => int($iops_rd),
4386         iops_wr => int($iops_wr),
4387         bps_max => int($bps_max),
4388         bps_rd_max => int($bps_rd_max),
4389         bps_wr_max => int($bps_wr_max),
4390         iops_max => int($iops_max),
4391         iops_rd_max => int($iops_rd_max),
4392         iops_wr_max => int($iops_wr_max),
4393         bps_max_length => int($bps_max_length),
4394         bps_rd_max_length => int($bps_rd_max_length),
4395         bps_wr_max_length => int($bps_wr_max_length),
4396         iops_max_length => int($iops_max_length),
4397         iops_rd_max_length => int($iops_rd_max_length),
4398         iops_wr_max_length => int($iops_wr_max_length),
4399     );
4400
4401 }
4402
4403 sub qemu_block_resize {
4404     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4405
4406     my $running = check_running($vmid);
4407
4408     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4409
4410     return if !$running;
4411
4412     my $padding = (1024 - $size % 1024) % 1024;
4413     $size = $size + $padding;
4414
4415     mon_cmd(
4416         $vmid,
4417         "block_resize",
4418         device => $deviceid,
4419         size => int($size),
4420         timeout => 60,
4421     );
4422 }
4423
4424 sub qemu_volume_snapshot {
4425     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4426
4427     my $running = check_running($vmid);
4428
4429     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4430         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4431     } else {
4432         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4433     }
4434 }
4435
4436 sub qemu_volume_snapshot_delete {
4437     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4438
4439     my $running = check_running($vmid);
4440
4441     if($running) {
4442
4443         $running = undef;
4444         my $conf = PVE::QemuConfig->load_config($vmid);
4445         PVE::QemuConfig->foreach_volume($conf, sub {
4446             my ($ds, $drive) = @_;
4447             $running = 1 if $drive->{file} eq $volid;
4448         });
4449     }
4450
4451     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4452         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4453     } else {
4454         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4455     }
4456 }
4457
4458 sub set_migration_caps {
4459     my ($vmid, $savevm) = @_;
4460
4461     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4462
4463     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4464     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4465
4466     my $cap_ref = [];
4467
4468     my $enabled_cap = {
4469         "auto-converge" => 1,
4470         "xbzrle" => 1,
4471         "x-rdma-pin-all" => 0,
4472         "zero-blocks" => 0,
4473         "compress" => 0,
4474         "dirty-bitmaps" => $dirty_bitmaps,
4475     };
4476
4477     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4478
4479     for my $supported_capability (@$supported_capabilities) {
4480         push @$cap_ref, {
4481             capability => $supported_capability->{capability},
4482             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4483         };
4484     }
4485
4486     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4487 }
4488
4489 sub foreach_volid {
4490     my ($conf, $func, @param) = @_;
4491
4492     my $volhash = {};
4493
4494     my $test_volid = sub {
4495         my ($key, $drive, $snapname) = @_;
4496
4497         my $volid = $drive->{file};
4498         return if !$volid;
4499
4500         $volhash->{$volid}->{cdrom} //= 1;
4501         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4502
4503         my $replicate = $drive->{replicate} // 1;
4504         $volhash->{$volid}->{replicate} //= 0;
4505         $volhash->{$volid}->{replicate} = 1 if $replicate;
4506
4507         $volhash->{$volid}->{shared} //= 0;
4508         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4509
4510         $volhash->{$volid}->{referenced_in_config} //= 0;
4511         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4512
4513         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4514             if defined($snapname);
4515
4516         my $size = $drive->{size};
4517         $volhash->{$volid}->{size} //= $size if $size;
4518
4519         $volhash->{$volid}->{is_vmstate} //= 0;
4520         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4521
4522         $volhash->{$volid}->{is_unused} //= 0;
4523         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4524
4525         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4526     };
4527
4528     my $include_opts = {
4529         extra_keys => ['vmstate'],
4530         include_unused => 1,
4531     };
4532
4533     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4534     foreach my $snapname (keys %{$conf->{snapshots}}) {
4535         my $snap = $conf->{snapshots}->{$snapname};
4536         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4537     }
4538
4539     foreach my $volid (keys %$volhash) {
4540         &$func($volid, $volhash->{$volid}, @param);
4541     }
4542 }
4543
4544 my $fast_plug_option = {
4545     'lock' => 1,
4546     'name' => 1,
4547     'onboot' => 1,
4548     'shares' => 1,
4549     'startup' => 1,
4550     'description' => 1,
4551     'protection' => 1,
4552     'vmstatestorage' => 1,
4553     'hookscript' => 1,
4554     'tags' => 1,
4555 };
4556
4557 # hotplug changes in [PENDING]
4558 # $selection hash can be used to only apply specified options, for
4559 # example: { cores => 1 } (only apply changed 'cores')
4560 # $errors ref is used to return error messages
4561 sub vmconfig_hotplug_pending {
4562     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4563
4564     my $defaults = load_defaults();
4565     my $arch = get_vm_arch($conf);
4566     my $machine_type = get_vm_machine($conf, undef, $arch);
4567
4568     # commit values which do not have any impact on running VM first
4569     # Note: those option cannot raise errors, we we do not care about
4570     # $selection and always apply them.
4571
4572     my $add_error = sub {
4573         my ($opt, $msg) = @_;
4574         $errors->{$opt} = "hotplug problem - $msg";
4575     };
4576
4577     my $changes = 0;
4578     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4579         if ($fast_plug_option->{$opt}) {
4580             $conf->{$opt} = $conf->{pending}->{$opt};
4581             delete $conf->{pending}->{$opt};
4582             $changes = 1;
4583         }
4584     }
4585
4586     if ($changes) {
4587         PVE::QemuConfig->write_config($vmid, $conf);
4588     }
4589
4590     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4591
4592     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4593     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4594     foreach my $opt (sort keys %$pending_delete_hash) {
4595         next if $selection && !$selection->{$opt};
4596         my $force = $pending_delete_hash->{$opt}->{force};
4597         eval {
4598             if ($opt eq 'hotplug') {
4599                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4600             } elsif ($opt eq 'tablet') {
4601                 die "skip\n" if !$hotplug_features->{usb};
4602                 if ($defaults->{tablet}) {
4603                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4604                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4605                         if $arch eq 'aarch64';
4606                 } else {
4607                     vm_deviceunplug($vmid, $conf, 'tablet');
4608                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4609                 }
4610             } elsif ($opt =~ m/^usb\d+/) {
4611                 die "skip\n";
4612                 # since we cannot reliably hot unplug usb devices we are disabling it
4613                 #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4614                 #vm_deviceunplug($vmid, $conf, $opt);
4615             } elsif ($opt eq 'vcpus') {
4616                 die "skip\n" if !$hotplug_features->{cpu};
4617                 qemu_cpu_hotplug($vmid, $conf, undef);
4618             } elsif ($opt eq 'balloon') {
4619                 # enable balloon device is not hotpluggable
4620                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4621                 # here we reset the ballooning value to memory
4622                 my $balloon = $conf->{memory} || $defaults->{memory};
4623                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4624             } elsif ($fast_plug_option->{$opt}) {
4625                 # do nothing
4626             } elsif ($opt =~ m/^net(\d+)$/) {
4627                 die "skip\n" if !$hotplug_features->{network};
4628                 vm_deviceunplug($vmid, $conf, $opt);
4629             } elsif (is_valid_drivename($opt)) {
4630                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4631                 vm_deviceunplug($vmid, $conf, $opt);
4632                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4633             } elsif ($opt =~ m/^memory$/) {
4634                 die "skip\n" if !$hotplug_features->{memory};
4635                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4636             } elsif ($opt eq 'cpuunits') {
4637                 $cgroup->change_cpu_shares(undef, 1024);
4638             } elsif ($opt eq 'cpulimit') {
4639                 $cgroup->change_cpu_quota(-1, 100000);
4640             } else {
4641                 die "skip\n";
4642             }
4643         };
4644         if (my $err = $@) {
4645             &$add_error($opt, $err) if $err ne "skip\n";
4646         } else {
4647             delete $conf->{$opt};
4648             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4649         }
4650     }
4651
4652     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4653     $apply_pending_cloudinit = sub {
4654         return if $apply_pending_cloudinit_done; # once is enough
4655         $apply_pending_cloudinit_done = 1; # once is enough
4656
4657         my ($key, $value) = @_;
4658
4659         my @cloudinit_opts = keys %$confdesc_cloudinit;
4660         foreach my $opt (keys %{$conf->{pending}}) {
4661             next if !grep { $_ eq $opt } @cloudinit_opts;
4662             $conf->{$opt} = delete $conf->{pending}->{$opt};
4663         }
4664
4665         my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4666         foreach my $opt (sort keys %$pending_delete_hash) {
4667             next if !grep { $_ eq $opt } @cloudinit_opts;
4668             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4669             delete $conf->{$opt};
4670         }
4671
4672         my $new_conf = { %$conf };
4673         $new_conf->{$key} = $value;
4674         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4675     };
4676
4677     foreach my $opt (keys %{$conf->{pending}}) {
4678         next if $selection && !$selection->{$opt};
4679         my $value = $conf->{pending}->{$opt};
4680         eval {
4681             if ($opt eq 'hotplug') {
4682                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4683             } elsif ($opt eq 'tablet') {
4684                 die "skip\n" if !$hotplug_features->{usb};
4685                 if ($value == 1) {
4686                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4687                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4688                         if $arch eq 'aarch64';
4689                 } elsif ($value == 0) {
4690                     vm_deviceunplug($vmid, $conf, 'tablet');
4691                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4692                 }
4693             } elsif ($opt =~ m/^usb\d+$/) {
4694                 die "skip\n";
4695                 # since we cannot reliably hot unplug usb devices we disable it for now
4696                 #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4697                 #my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4698                 #die "skip\n" if !$d;
4699                 #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4700             } elsif ($opt eq 'vcpus') {
4701                 die "skip\n" if !$hotplug_features->{cpu};
4702                 qemu_cpu_hotplug($vmid, $conf, $value);
4703             } elsif ($opt eq 'balloon') {
4704                 # enable/disable balloning device is not hotpluggable
4705                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4706                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4707                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4708
4709                 # allow manual ballooning if shares is set to zero
4710                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4711                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4712                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4713                 }
4714             } elsif ($opt =~ m/^net(\d+)$/) {
4715                 # some changes can be done without hotplug
4716                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4717                                     $vmid, $opt, $value, $arch, $machine_type);
4718             } elsif (is_valid_drivename($opt)) {
4719                 die "skip\n" if $opt eq 'efidisk0';
4720                 # some changes can be done without hotplug
4721                 my $drive = parse_drive($opt, $value);
4722                 if (drive_is_cloudinit($drive)) {
4723                     &$apply_pending_cloudinit($opt, $value);
4724                 }
4725                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4726                                      $vmid, $opt, $value, $arch, $machine_type);
4727             } elsif ($opt =~ m/^memory$/) { #dimms
4728                 die "skip\n" if !$hotplug_features->{memory};
4729                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4730             } elsif ($opt eq 'cpuunits') {
4731                 $cgroup->change_cpu_shares($conf->{pending}->{$opt}, 1024);
4732             } elsif ($opt eq 'cpulimit') {
4733                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4734                 $cgroup->change_cpu_quota($cpulimit, 100000);
4735             } else {
4736                 die "skip\n";  # skip non-hot-pluggable options
4737             }
4738         };
4739         if (my $err = $@) {
4740             &$add_error($opt, $err) if $err ne "skip\n";
4741         } else {
4742             $conf->{$opt} = $value;
4743             delete $conf->{pending}->{$opt};
4744         }
4745     }
4746
4747     PVE::QemuConfig->write_config($vmid, $conf);
4748 }
4749
4750 sub try_deallocate_drive {
4751     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4752
4753     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4754         my $volid = $drive->{file};
4755         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4756             my $sid = PVE::Storage::parse_volume_id($volid);
4757             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4758
4759             # check if the disk is really unused
4760             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4761                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4762             PVE::Storage::vdisk_free($storecfg, $volid);
4763             return 1;
4764         } else {
4765             # If vm is not owner of this disk remove from config
4766             return 1;
4767         }
4768     }
4769
4770     return;
4771 }
4772
4773 sub vmconfig_delete_or_detach_drive {
4774     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4775
4776     my $drive = parse_drive($opt, $conf->{$opt});
4777
4778     my $rpcenv = PVE::RPCEnvironment::get();
4779     my $authuser = $rpcenv->get_user();
4780
4781     if ($force) {
4782         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4783         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4784     } else {
4785         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4786     }
4787 }
4788
4789
4790
4791 sub vmconfig_apply_pending {
4792     my ($vmid, $conf, $storecfg, $errors) = @_;
4793
4794     my $add_apply_error = sub {
4795         my ($opt, $msg) = @_;
4796         my $err_msg = "unable to apply pending change $opt : $msg";
4797         $errors->{$opt} = $err_msg;
4798         warn $err_msg;
4799     };
4800
4801     # cold plug
4802
4803     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4804     foreach my $opt (sort keys %$pending_delete_hash) {
4805         my $force = $pending_delete_hash->{$opt}->{force};
4806         eval {
4807             if ($opt =~ m/^unused/) {
4808                 die "internal error";
4809             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4810                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4811             }
4812         };
4813         if (my $err = $@) {
4814             $add_apply_error->($opt, $err);
4815         } else {
4816             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4817             delete $conf->{$opt};
4818         }
4819     }
4820
4821     PVE::QemuConfig->cleanup_pending($conf);
4822
4823     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4824         next if $opt eq 'delete'; # just to be sure
4825         eval {
4826             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4827                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4828             }
4829         };
4830         if (my $err = $@) {
4831             $add_apply_error->($opt, $err);
4832         } else {
4833             $conf->{$opt} = delete $conf->{pending}->{$opt};
4834         }
4835     }
4836
4837     # write all changes at once to avoid unnecessary i/o
4838     PVE::QemuConfig->write_config($vmid, $conf);
4839 }
4840
4841 sub vmconfig_update_net {
4842     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4843
4844     my $newnet = parse_net($value);
4845
4846     if ($conf->{$opt}) {
4847         my $oldnet = parse_net($conf->{$opt});
4848
4849         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4850             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4851             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4852             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4853
4854             # for non online change, we try to hot-unplug
4855             die "skip\n" if !$hotplug;
4856             vm_deviceunplug($vmid, $conf, $opt);
4857         } else {
4858
4859             die "internal error" if $opt !~ m/net(\d+)/;
4860             my $iface = "tap${vmid}i$1";
4861
4862             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4863                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4864                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4865                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4866                 PVE::Network::tap_unplug($iface);
4867
4868                 if ($have_sdn) {
4869                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4870                 } else {
4871                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4872                 }
4873             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4874                 # Rate can be applied on its own but any change above needs to
4875                 # include the rate in tap_plug since OVS resets everything.
4876                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4877             }
4878
4879             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4880                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4881             }
4882
4883             return 1;
4884         }
4885     }
4886
4887     if ($hotplug) {
4888         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4889     } else {
4890         die "skip\n";
4891     }
4892 }
4893
4894 sub vmconfig_update_disk {
4895     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4896
4897     my $drive = parse_drive($opt, $value);
4898
4899     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
4900         my $media = $drive->{media} || 'disk';
4901         my $oldmedia = $old_drive->{media} || 'disk';
4902         die "unable to change media type\n" if $media ne $oldmedia;
4903
4904         if (!drive_is_cdrom($old_drive)) {
4905
4906             if ($drive->{file} ne $old_drive->{file}) {
4907
4908                 die "skip\n" if !$hotplug;
4909
4910                 # unplug and register as unused
4911                 vm_deviceunplug($vmid, $conf, $opt);
4912                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4913
4914             } else {
4915                 # update existing disk
4916
4917                 # skip non hotpluggable value
4918                 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4919                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4920                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4921                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
4922                     safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
4923                     die "skip\n";
4924                 }
4925
4926                 # apply throttle
4927                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4928                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4929                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4930                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4931                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4932                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4933                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4934                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4935                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4936                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4937                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4938                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4939                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4940                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4941                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4942                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4943                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4944                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4945
4946                     qemu_block_set_io_throttle(
4947                         $vmid,"drive-$opt",
4948                         ($drive->{mbps} || 0)*1024*1024,
4949                         ($drive->{mbps_rd} || 0)*1024*1024,
4950                         ($drive->{mbps_wr} || 0)*1024*1024,
4951                          $drive->{iops} || 0,
4952                          $drive->{iops_rd} || 0,
4953                          $drive->{iops_wr} || 0,
4954                         ($drive->{mbps_max} || 0)*1024*1024,
4955                         ($drive->{mbps_rd_max} || 0)*1024*1024,
4956                         ($drive->{mbps_wr_max} || 0)*1024*1024,
4957                          $drive->{iops_max} || 0,
4958                          $drive->{iops_rd_max} || 0,
4959                          $drive->{iops_wr_max} || 0,
4960                          $drive->{bps_max_length} || 1,
4961                          $drive->{bps_rd_max_length} || 1,
4962                          $drive->{bps_wr_max_length} || 1,
4963                          $drive->{iops_max_length} || 1,
4964                          $drive->{iops_rd_max_length} || 1,
4965                          $drive->{iops_wr_max_length} || 1,
4966                     );
4967
4968                 }
4969
4970                 return 1;
4971             }
4972
4973         } else { # cdrom
4974
4975             if ($drive->{file} eq 'none') {
4976                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4977                 if (drive_is_cloudinit($old_drive)) {
4978                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4979                 }
4980             } else {
4981                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4982
4983                 # force eject if locked
4984                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4985
4986                 if ($path) {
4987                     mon_cmd($vmid, "blockdev-change-medium",
4988                         id => "$opt", filename => "$path");
4989                 }
4990             }
4991
4992             return 1;
4993         }
4994     }
4995
4996     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4997     # hotplug new disks
4998     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4999     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5000 }
5001
5002 # called in locked context by incoming migration
5003 sub vm_migrate_get_nbd_disks {
5004     my ($storecfg, $conf, $replicated_volumes) = @_;
5005
5006     my $local_volumes = {};
5007     PVE::QemuConfig->foreach_volume($conf, sub {
5008         my ($ds, $drive) = @_;
5009
5010         return if drive_is_cdrom($drive);
5011
5012         my $volid = $drive->{file};
5013
5014         return if !$volid;
5015
5016         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5017
5018         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5019         return if $scfg->{shared};
5020
5021         # replicated disks re-use existing state via bitmap
5022         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5023         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
5024     });
5025     return $local_volumes;
5026 }
5027
5028 # called in locked context by incoming migration
5029 sub vm_migrate_alloc_nbd_disks {
5030     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5031
5032     my $format = undef;
5033
5034     my $nbd = {};
5035     foreach my $opt (sort keys %$source_volumes) {
5036         my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
5037
5038         if ($use_existing) {
5039             $nbd->{$opt}->{drivestr} = print_drive($drive);
5040             $nbd->{$opt}->{volid} = $volid;
5041             $nbd->{$opt}->{replicated} = 1;
5042             next;
5043         }
5044
5045         # If a remote storage is specified and the format of the original
5046         # volume is not available there, fall back to the default format.
5047         # Otherwise use the same format as the original.
5048         if (!$storagemap->{identity}) {
5049             $storeid = map_storage($storagemap, $storeid);
5050             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5051             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5052             my $fileFormat = qemu_img_format($scfg, $volname);
5053             $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
5054         } else {
5055             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5056             $format = qemu_img_format($scfg, $volname);
5057         }
5058
5059         my $size = $drive->{size} / 1024;
5060         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5061         my $newdrive = $drive;
5062         $newdrive->{format} = $format;
5063         $newdrive->{file} = $newvolid;
5064         my $drivestr = print_drive($newdrive);
5065         $nbd->{$opt}->{drivestr} = $drivestr;
5066         $nbd->{$opt}->{volid} = $newvolid;
5067     }
5068
5069     return $nbd;
5070 }
5071
5072 # see vm_start_nolock for parameters, additionally:
5073 # migrate_opts:
5074 #   storagemap = parsed storage map for allocating NBD disks
5075 sub vm_start {
5076     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5077
5078     return PVE::QemuConfig->lock_config($vmid, sub {
5079         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5080
5081         die "you can't start a vm if it's a template\n"
5082             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5083
5084         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5085         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5086
5087         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5088
5089         if ($has_backup_lock && $running) {
5090             # a backup is currently running, attempt to start the guest in the
5091             # existing QEMU instance
5092             return vm_resume($vmid);
5093         }
5094
5095         PVE::QemuConfig->check_lock($conf)
5096             if !($params->{skiplock} || $has_suspended_lock);
5097
5098         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5099
5100         die "VM $vmid already running\n" if $running;
5101
5102         if (my $storagemap = $migrate_opts->{storagemap}) {
5103             my $replicated = $migrate_opts->{replicated_volumes};
5104             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5105             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5106
5107             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5108                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5109             }
5110         }
5111
5112         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5113     });
5114 }
5115
5116
5117 # params:
5118 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5119 #   skiplock => 0/1, skip checking for config lock
5120 #   skiptemplate => 0/1, skip checking whether VM is template
5121 #   forcemachine => to force Qemu machine (rollback/migration)
5122 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5123 #   timeout => in seconds
5124 #   paused => start VM in paused state (backup)
5125 #   resume => resume from hibernation
5126 #   pbs-backing => {
5127 #      sata0 => {
5128 #         repository
5129 #         snapshot
5130 #         keyfile
5131 #         archive
5132 #      },
5133 #      virtio2 => ...
5134 #   }
5135 # migrate_opts:
5136 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5137 #   migratedfrom => source node
5138 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5139 #   network => CIDR of migration network
5140 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5141 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5142 #   replicated_volumes = which volids should be re-used with bitmaps for nbd migration
5143 sub vm_start_nolock {
5144     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5145
5146     my $statefile = $params->{statefile};
5147     my $resume = $params->{resume};
5148
5149     my $migratedfrom = $migrate_opts->{migratedfrom};
5150     my $migration_type = $migrate_opts->{type};
5151
5152     my $res = {};
5153
5154     # clean up leftover reboot request files
5155     eval { clear_reboot_request($vmid); };
5156     warn $@ if $@;
5157
5158     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5159         vmconfig_apply_pending($vmid, $conf, $storecfg);
5160         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5161     }
5162
5163     # don't regenerate the ISO if the VM is started as part of a live migration
5164     # this way we can reuse the old ISO with the correct config
5165     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid) if !$migratedfrom;
5166
5167     my $defaults = load_defaults();
5168
5169     # set environment variable useful inside network script
5170     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5171
5172     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5173
5174     my $forcemachine = $params->{forcemachine};
5175     my $forcecpu = $params->{forcecpu};
5176     if ($resume) {
5177         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5178         $forcemachine = $conf->{runningmachine};
5179         $forcecpu = $conf->{runningcpu};
5180         print "Resuming suspended VM\n";
5181     }
5182
5183     my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid,
5184         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5185
5186     my $migration_ip;
5187     my $get_migration_ip = sub {
5188         my ($nodename) = @_;
5189
5190         return $migration_ip if defined($migration_ip);
5191
5192         my $cidr = $migrate_opts->{network};
5193
5194         if (!defined($cidr)) {
5195             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5196             $cidr = $dc_conf->{migration}->{network};
5197         }
5198
5199         if (defined($cidr)) {
5200             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5201
5202             die "could not get IP: no address configured on local " .
5203                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5204
5205             die "could not get IP: multiple addresses configured on local " .
5206                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5207
5208             $migration_ip = @$ips[0];
5209         }
5210
5211         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5212             if !defined($migration_ip);
5213
5214         return $migration_ip;
5215     };
5216
5217     my $migrate_uri;
5218     if ($statefile) {
5219         if ($statefile eq 'tcp') {
5220             my $localip = "localhost";
5221             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5222             my $nodename = nodename();
5223
5224             if (!defined($migration_type)) {
5225                 if (defined($datacenterconf->{migration}->{type})) {
5226                     $migration_type = $datacenterconf->{migration}->{type};
5227                 } else {
5228                     $migration_type = 'secure';
5229                 }
5230             }
5231
5232             if ($migration_type eq 'insecure') {
5233                 $localip = $get_migration_ip->($nodename);
5234                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5235             }
5236
5237             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5238             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5239             $migrate_uri = "tcp:${localip}:${migrate_port}";
5240             push @$cmd, '-incoming', $migrate_uri;
5241             push @$cmd, '-S';
5242
5243         } elsif ($statefile eq 'unix') {
5244             # should be default for secure migrations as a ssh TCP forward
5245             # tunnel is not deterministic reliable ready and fails regurarly
5246             # to set up in time, so use UNIX socket forwards
5247             my $socket_addr = "/run/qemu-server/$vmid.migrate";
5248             unlink $socket_addr;
5249
5250             $migrate_uri = "unix:$socket_addr";
5251
5252             push @$cmd, '-incoming', $migrate_uri;
5253             push @$cmd, '-S';
5254
5255         } elsif (-e $statefile) {
5256             push @$cmd, '-loadstate', $statefile;
5257         } else {
5258             my $statepath = PVE::Storage::path($storecfg, $statefile);
5259             push @$vollist, $statefile;
5260             push @$cmd, '-loadstate', $statepath;
5261         }
5262     } elsif ($params->{paused}) {
5263         push @$cmd, '-S';
5264     }
5265
5266     # host pci devices
5267     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5268       my $d = parse_hostpci($conf->{"hostpci$i"});
5269       next if !$d;
5270       my $pcidevices = $d->{pciid};
5271       foreach my $pcidevice (@$pcidevices) {
5272             my $pciid = $pcidevice->{id};
5273
5274             my $info = PVE::SysFSTools::pci_device_info("$pciid");
5275             die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5276             die "no pci device info for device '$pciid'\n" if !$info;
5277
5278             if ($d->{mdev}) {
5279                 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5280                 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5281             } else {
5282                 die "can't unbind/bind PCI group to VFIO '$pciid'\n"
5283                     if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5284                 die "can't reset PCI device '$pciid'\n"
5285                     if $info->{has_fl_reset} && !PVE::SysFSTools::pci_dev_reset($info);
5286             }
5287       }
5288     }
5289
5290     PVE::Storage::activate_volumes($storecfg, $vollist);
5291
5292     eval {
5293         run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5294             outfunc => sub {}, errfunc => sub {});
5295     };
5296     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5297     # timeout should be more than enough here...
5298     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5299
5300     my $cpuunits = get_cpuunits($conf);
5301
5302     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5303     my %run_params = (
5304         timeout => $statefile ? undef : $start_timeout,
5305         umask => 0077,
5306         noerr => 1,
5307     );
5308
5309     # when migrating, prefix QEMU output so other side can pick up any
5310     # errors that might occur and show the user
5311     if ($migratedfrom) {
5312         $run_params{quiet} = 1;
5313         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5314     }
5315
5316     my %properties = (
5317         Slice => 'qemu.slice',
5318         KillMode => 'process',
5319         SendSIGKILL => 0,
5320         TimeoutStopUSec => ULONG_MAX, # infinity
5321     );
5322
5323     if (PVE::CGroup::cgroup_mode() == 2) {
5324         $cpuunits = 10000 if $cpuunits >= 10000; # else we get an error
5325         $properties{CPUWeight} = $cpuunits;
5326     } else {
5327         $properties{CPUShares} = $cpuunits;
5328     }
5329
5330     if (my $cpulimit = $conf->{cpulimit}) {
5331         $properties{CPUQuota} = int($cpulimit * 100);
5332     }
5333     $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5334
5335     my $run_qemu = sub {
5336         PVE::Tools::run_fork sub {
5337             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5338
5339             my $exitcode = run_command($cmd, %run_params);
5340             die "QEMU exited with code $exitcode\n" if $exitcode;
5341         };
5342     };
5343
5344     if ($conf->{hugepages}) {
5345
5346         my $code = sub {
5347             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5348             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5349
5350             PVE::QemuServer::Memory::hugepages_mount();
5351             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5352
5353             eval { $run_qemu->() };
5354             if (my $err = $@) {
5355                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5356                     if !$conf->{keephugepages};
5357                 die $err;
5358             }
5359
5360             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5361                 if !$conf->{keephugepages};
5362         };
5363         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5364
5365     } else {
5366         eval { $run_qemu->() };
5367     }
5368
5369     if (my $err = $@) {
5370         # deactivate volumes if start fails
5371         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5372         die "start failed: $err";
5373     }
5374
5375     print "migration listens on $migrate_uri\n" if $migrate_uri;
5376     $res->{migrate_uri} = $migrate_uri;
5377
5378     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5379         eval { mon_cmd($vmid, "cont"); };
5380         warn $@ if $@;
5381     }
5382
5383     #start nbd server for storage migration
5384     if (my $nbd = $migrate_opts->{nbd}) {
5385         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5386
5387         my $migrate_storage_uri;
5388         # nbd_protocol_version > 0 for unix socket support
5389         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5390             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5391             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5392             $migrate_storage_uri = "nbd:unix:$socket_path";
5393         } else {
5394             my $nodename = nodename();
5395             my $localip = $get_migration_ip->($nodename);
5396             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5397             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5398
5399             mon_cmd($vmid, "nbd-server-start", addr => {
5400                 type => 'inet',
5401                 data => {
5402                     host => "${localip}",
5403                     port => "${storage_migrate_port}",
5404                 },
5405             });
5406             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5407             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5408         }
5409
5410         $res->{migrate_storage_uri} = $migrate_storage_uri;
5411
5412         foreach my $opt (sort keys %$nbd) {
5413             my $drivestr = $nbd->{$opt}->{drivestr};
5414             my $volid = $nbd->{$opt}->{volid};
5415             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5416             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5417             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5418             print "re-using replicated volume: $opt - $volid\n"
5419                 if $nbd->{$opt}->{replicated};
5420
5421             $res->{drives}->{$opt} = $nbd->{$opt};
5422             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5423         }
5424     }
5425
5426     if ($migratedfrom) {
5427         eval {
5428             set_migration_caps($vmid);
5429         };
5430         warn $@ if $@;
5431
5432         if ($spice_port) {
5433             print "spice listens on port $spice_port\n";
5434             $res->{spice_port} = $spice_port;
5435             if ($migrate_opts->{spice_ticket}) {
5436                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5437                     $migrate_opts->{spice_ticket});
5438                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5439             }
5440         }
5441
5442     } else {
5443         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5444             if !$statefile && $conf->{balloon};
5445
5446         foreach my $opt (keys %$conf) {
5447             next if $opt !~  m/^net\d+$/;
5448             my $nicconf = parse_net($conf->{$opt});
5449             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5450         }
5451     }
5452
5453     mon_cmd($vmid, 'qom-set',
5454                 path => "machine/peripheral/balloon0",
5455                 property => "guest-stats-polling-interval",
5456                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5457
5458     if ($resume) {
5459         print "Resumed VM, removing state\n";
5460         if (my $vmstate = $conf->{vmstate}) {
5461             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5462             PVE::Storage::vdisk_free($storecfg, $vmstate);
5463         }
5464         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5465         PVE::QemuConfig->write_config($vmid, $conf);
5466     }
5467
5468     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5469
5470     return $res;
5471 }
5472
5473 sub vm_commandline {
5474     my ($storecfg, $vmid, $snapname) = @_;
5475
5476     my $conf = PVE::QemuConfig->load_config($vmid);
5477     my $forcemachine;
5478     my $forcecpu;
5479
5480     if ($snapname) {
5481         my $snapshot = $conf->{snapshots}->{$snapname};
5482         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5483
5484         # check for machine or CPU overrides in snapshot
5485         $forcemachine = $snapshot->{runningmachine};
5486         $forcecpu = $snapshot->{runningcpu};
5487
5488         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5489
5490         $conf = $snapshot;
5491     }
5492
5493     my $defaults = load_defaults();
5494
5495     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults,
5496         $forcemachine, $forcecpu);
5497
5498     return PVE::Tools::cmd2string($cmd);
5499 }
5500
5501 sub vm_reset {
5502     my ($vmid, $skiplock) = @_;
5503
5504     PVE::QemuConfig->lock_config($vmid, sub {
5505
5506         my $conf = PVE::QemuConfig->load_config($vmid);
5507
5508         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5509
5510         mon_cmd($vmid, "system_reset");
5511     });
5512 }
5513
5514 sub get_vm_volumes {
5515     my ($conf) = @_;
5516
5517     my $vollist = [];
5518     foreach_volid($conf, sub {
5519         my ($volid, $attr) = @_;
5520
5521         return if $volid =~ m|^/|;
5522
5523         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5524         return if !$sid;
5525
5526         push @$vollist, $volid;
5527     });
5528
5529     return $vollist;
5530 }
5531
5532 sub vm_stop_cleanup {
5533     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5534
5535     eval {
5536
5537         if (!$keepActive) {
5538             my $vollist = get_vm_volumes($conf);
5539             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5540         }
5541
5542         foreach my $ext (qw(mon qmp pid vnc qga)) {
5543             unlink "/var/run/qemu-server/${vmid}.$ext";
5544         }
5545
5546         if ($conf->{ivshmem}) {
5547             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5548             # just delete it for now, VMs which have this already open do not
5549             # are affected, but new VMs will get a separated one. If this
5550             # becomes an issue we either add some sort of ref-counting or just
5551             # add a "don't delete on stop" flag to the ivshmem format.
5552             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5553         }
5554
5555         foreach my $key (keys %$conf) {
5556             next if $key !~ m/^hostpci(\d+)$/;
5557             my $hostpciindex = $1;
5558             my $d = parse_hostpci($conf->{$key});
5559             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5560
5561             foreach my $pci (@{$d->{pciid}}) {
5562                 my $pciid = $pci->{id};
5563                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5564             }
5565         }
5566
5567         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5568     };
5569     warn $@ if $@; # avoid errors - just warn
5570 }
5571
5572 # call only in locked context
5573 sub _do_vm_stop {
5574     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5575
5576     my $pid = check_running($vmid, $nocheck);
5577     return if !$pid;
5578
5579     my $conf;
5580     if (!$nocheck) {
5581         $conf = PVE::QemuConfig->load_config($vmid);
5582         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5583         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5584             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5585             $timeout = $opts->{down} if $opts->{down};
5586         }
5587         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5588     }
5589
5590     eval {
5591         if ($shutdown) {
5592             if (defined($conf) && get_qga_key($conf, 'enabled')) {
5593                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5594             } else {
5595                 mon_cmd($vmid, "system_powerdown");
5596             }
5597         } else {
5598             mon_cmd($vmid, "quit");
5599         }
5600     };
5601     my $err = $@;
5602
5603     if (!$err) {
5604         $timeout = 60 if !defined($timeout);
5605
5606         my $count = 0;
5607         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5608             $count++;
5609             sleep 1;
5610         }
5611
5612         if ($count >= $timeout) {
5613             if ($force) {
5614                 warn "VM still running - terminating now with SIGTERM\n";
5615                 kill 15, $pid;
5616             } else {
5617                 die "VM quit/powerdown failed - got timeout\n";
5618             }
5619         } else {
5620             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5621             return;
5622         }
5623     } else {
5624        if (!check_running($vmid, $nocheck)) {
5625             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5626             return;
5627        }
5628        if ($force) {
5629             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5630             kill 15, $pid;
5631         } else {
5632             die "VM quit/powerdown failed\n";
5633         }
5634     }
5635
5636     # wait again
5637     $timeout = 10;
5638
5639     my $count = 0;
5640     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5641         $count++;
5642         sleep 1;
5643     }
5644
5645     if ($count >= $timeout) {
5646         warn "VM still running - terminating now with SIGKILL\n";
5647         kill 9, $pid;
5648         sleep 1;
5649     }
5650
5651     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5652 }
5653
5654 # Note: use $nocheck to skip tests if VM configuration file exists.
5655 # We need that when migration VMs to other nodes (files already moved)
5656 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5657 sub vm_stop {
5658     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5659
5660     $force = 1 if !defined($force) && !$shutdown;
5661
5662     if ($migratedfrom){
5663         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5664         kill 15, $pid if $pid;
5665         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5666         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5667         return;
5668     }
5669
5670     PVE::QemuConfig->lock_config($vmid, sub {
5671         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5672    });
5673 }
5674
5675 sub vm_reboot {
5676     my ($vmid, $timeout) = @_;
5677
5678     PVE::QemuConfig->lock_config($vmid, sub {
5679         eval {
5680
5681             # only reboot if running, as qmeventd starts it again on a stop event
5682             return if !check_running($vmid);
5683
5684             create_reboot_request($vmid);
5685
5686             my $storecfg = PVE::Storage::config();
5687             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5688
5689         };
5690         if (my $err = $@) {
5691             # avoid that the next normal shutdown will be confused for a reboot
5692             clear_reboot_request($vmid);
5693             die $err;
5694         }
5695    });
5696 }
5697
5698 # note: if using the statestorage parameter, the caller has to check privileges
5699 sub vm_suspend {
5700     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5701
5702     my $conf;
5703     my $path;
5704     my $storecfg;
5705     my $vmstate;
5706
5707     PVE::QemuConfig->lock_config($vmid, sub {
5708
5709         $conf = PVE::QemuConfig->load_config($vmid);
5710
5711         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5712         PVE::QemuConfig->check_lock($conf)
5713             if !($skiplock || $is_backing_up);
5714
5715         die "cannot suspend to disk during backup\n"
5716             if $is_backing_up && $includestate;
5717
5718         if ($includestate) {
5719             $conf->{lock} = 'suspending';
5720             my $date = strftime("%Y-%m-%d", localtime(time()));
5721             $storecfg = PVE::Storage::config();
5722             if (!$statestorage) {
5723                 $statestorage = find_vmstate_storage($conf, $storecfg);
5724                 # check permissions for the storage
5725                 my $rpcenv = PVE::RPCEnvironment::get();
5726                 if ($rpcenv->{type} ne 'cli') {
5727                     my $authuser = $rpcenv->get_user();
5728                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5729                 }
5730             }
5731
5732
5733             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
5734                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5735             $path = PVE::Storage::path($storecfg, $vmstate);
5736             PVE::QemuConfig->write_config($vmid, $conf);
5737         } else {
5738             mon_cmd($vmid, "stop");
5739         }
5740     });
5741
5742     if ($includestate) {
5743         # save vm state
5744         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5745
5746         eval {
5747             set_migration_caps($vmid, 1);
5748             mon_cmd($vmid, "savevm-start", statefile => $path);
5749             for(;;) {
5750                 my $state = mon_cmd($vmid, "query-savevm");
5751                 if (!$state->{status}) {
5752                     die "savevm not active\n";
5753                 } elsif ($state->{status} eq 'active') {
5754                     sleep(1);
5755                     next;
5756                 } elsif ($state->{status} eq 'completed') {
5757                     print "State saved, quitting\n";
5758                     last;
5759                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5760                     die "query-savevm failed with error '$state->{error}'\n"
5761                 } else {
5762                     die "query-savevm returned status '$state->{status}'\n";
5763                 }
5764             }
5765         };
5766         my $err = $@;
5767
5768         PVE::QemuConfig->lock_config($vmid, sub {
5769             $conf = PVE::QemuConfig->load_config($vmid);
5770             if ($err) {
5771                 # cleanup, but leave suspending lock, to indicate something went wrong
5772                 eval {
5773                     mon_cmd($vmid, "savevm-end");
5774                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5775                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5776                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
5777                     PVE::QemuConfig->write_config($vmid, $conf);
5778                 };
5779                 warn $@ if $@;
5780                 die $err;
5781             }
5782
5783             die "lock changed unexpectedly\n"
5784                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5785
5786             mon_cmd($vmid, "quit");
5787             $conf->{lock} = 'suspended';
5788             PVE::QemuConfig->write_config($vmid, $conf);
5789         });
5790     }
5791 }
5792
5793 sub vm_resume {
5794     my ($vmid, $skiplock, $nocheck) = @_;
5795
5796     PVE::QemuConfig->lock_config($vmid, sub {
5797         my $res = mon_cmd($vmid, 'query-status');
5798         my $resume_cmd = 'cont';
5799         my $reset = 0;
5800
5801         if ($res->{status}) {
5802             return if $res->{status} eq 'running'; # job done, go home
5803             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
5804             $reset = 1 if $res->{status} eq 'shutdown';
5805         }
5806
5807         if (!$nocheck) {
5808
5809             my $conf = PVE::QemuConfig->load_config($vmid);
5810
5811             PVE::QemuConfig->check_lock($conf)
5812                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5813         }
5814
5815         if ($reset) {
5816             # required if a VM shuts down during a backup and we get a resume
5817             # request before the backup finishes for example
5818             mon_cmd($vmid, "system_reset");
5819         }
5820         mon_cmd($vmid, $resume_cmd);
5821     });
5822 }
5823
5824 sub vm_sendkey {
5825     my ($vmid, $skiplock, $key) = @_;
5826
5827     PVE::QemuConfig->lock_config($vmid, sub {
5828
5829         my $conf = PVE::QemuConfig->load_config($vmid);
5830
5831         # there is no qmp command, so we use the human monitor command
5832         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5833         die $res if $res ne '';
5834     });
5835 }
5836
5837 # vzdump restore implementaion
5838
5839 sub tar_archive_read_firstfile {
5840     my $archive = shift;
5841
5842     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5843
5844     # try to detect archive type first
5845     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5846         die "unable to open file '$archive'\n";
5847     my $firstfile = <$fh>;
5848     kill 15, $pid;
5849     close $fh;
5850
5851     die "ERROR: archive contaions no data\n" if !$firstfile;
5852     chomp $firstfile;
5853
5854     return $firstfile;
5855 }
5856
5857 sub tar_restore_cleanup {
5858     my ($storecfg, $statfile) = @_;
5859
5860     print STDERR "starting cleanup\n";
5861
5862     if (my $fd = IO::File->new($statfile, "r")) {
5863         while (defined(my $line = <$fd>)) {
5864             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5865                 my $volid = $2;
5866                 eval {
5867                     if ($volid =~ m|^/|) {
5868                         unlink $volid || die 'unlink failed\n';
5869                     } else {
5870                         PVE::Storage::vdisk_free($storecfg, $volid);
5871                     }
5872                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5873                 };
5874                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5875             } else {
5876                 print STDERR "unable to parse line in statfile - $line";
5877             }
5878         }
5879         $fd->close();
5880     }
5881 }
5882
5883 sub restore_file_archive {
5884     my ($archive, $vmid, $user, $opts) = @_;
5885
5886     return restore_vma_archive($archive, $vmid, $user, $opts)
5887         if $archive eq '-';
5888
5889     my $info = PVE::Storage::archive_info($archive);
5890     my $format = $opts->{format} // $info->{format};
5891     my $comp = $info->{compression};
5892
5893     # try to detect archive format
5894     if ($format eq 'tar') {
5895         return restore_tar_archive($archive, $vmid, $user, $opts);
5896     } else {
5897         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5898     }
5899 }
5900
5901 # hepler to remove disks that will not be used after restore
5902 my $restore_cleanup_oldconf = sub {
5903     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
5904
5905     PVE::QemuConfig->foreach_volume($oldconf, sub {
5906         my ($ds, $drive) = @_;
5907
5908         return if drive_is_cdrom($drive, 1);
5909
5910         my $volid = $drive->{file};
5911         return if !$volid || $volid =~ m|^/|;
5912
5913         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
5914         return if !$path || !$owner || ($owner != $vmid);
5915
5916         # Note: only delete disk we want to restore
5917         # other volumes will become unused
5918         if ($virtdev_hash->{$ds}) {
5919             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
5920             if (my $err = $@) {
5921                 warn $err;
5922             }
5923         }
5924     });
5925
5926     # delete vmstate files, after the restore we have no snapshots anymore
5927     foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5928         my $snap = $oldconf->{snapshots}->{$snapname};
5929         if ($snap->{vmstate}) {
5930             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
5931             if (my $err = $@) {
5932                 warn $err;
5933             }
5934         }
5935     }
5936 };
5937
5938 # Helper to parse vzdump backup device hints
5939 #
5940 # $rpcenv: Environment, used to ckeck storage permissions
5941 # $user: User ID, to check storage permissions
5942 # $storecfg: Storage configuration
5943 # $fh: the file handle for reading the configuration
5944 # $devinfo: should contain device sizes for all backu-up'ed devices
5945 # $options: backup options (pool, default storage)
5946 #
5947 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
5948 my $parse_backup_hints = sub {
5949     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
5950
5951     my $virtdev_hash = {};
5952
5953     while (defined(my $line = <$fh>)) {
5954         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5955             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5956             die "archive does not contain data for drive '$virtdev'\n"
5957                 if !$devinfo->{$devname};
5958
5959             if (defined($options->{storage})) {
5960                 $storeid = $options->{storage} || 'local';
5961             } elsif (!$storeid) {
5962                 $storeid = 'local';
5963             }
5964             $format = 'raw' if !$format;
5965             $devinfo->{$devname}->{devname} = $devname;
5966             $devinfo->{$devname}->{virtdev} = $virtdev;
5967             $devinfo->{$devname}->{format} = $format;
5968             $devinfo->{$devname}->{storeid} = $storeid;
5969
5970             # check permission on storage
5971             my $pool = $options->{pool}; # todo: do we need that?
5972             if ($user ne 'root@pam') {
5973                 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5974             }
5975
5976             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5977         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
5978             my $virtdev = $1;
5979             my $drive = parse_drive($virtdev, $2);
5980             if (drive_is_cloudinit($drive)) {
5981                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5982                 $storeid = $options->{storage} if defined ($options->{storage});
5983                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5984                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
5985
5986                 $virtdev_hash->{$virtdev} = {
5987                     format => $format,
5988                     storeid => $storeid,
5989                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
5990                     is_cloudinit => 1,
5991                 };
5992             }
5993         }
5994     }
5995
5996     return $virtdev_hash;
5997 };
5998
5999 # Helper to allocate and activate all volumes required for a restore
6000 #
6001 # $storecfg: Storage configuration
6002 # $virtdev_hash: as returned by parse_backup_hints()
6003 #
6004 # Returns: { $virtdev => $volid }
6005 my $restore_allocate_devices = sub {
6006     my ($storecfg, $virtdev_hash, $vmid) = @_;
6007
6008     my $map = {};
6009     foreach my $virtdev (sort keys %$virtdev_hash) {
6010         my $d = $virtdev_hash->{$virtdev};
6011         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6012         my $storeid = $d->{storeid};
6013         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6014
6015         # test if requested format is supported
6016         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6017         my $supported = grep { $_ eq $d->{format} } @$validFormats;
6018         $d->{format} = $defFormat if !$supported;
6019
6020         my $name;
6021         if ($d->{is_cloudinit}) {
6022             $name = "vm-$vmid-cloudinit";
6023             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6024             if ($scfg->{path}) {
6025                 $name .= ".$d->{format}";
6026             }
6027         }
6028
6029         my $volid = PVE::Storage::vdisk_alloc(
6030             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6031
6032         print STDERR "new volume ID is '$volid'\n";
6033         $d->{volid} = $volid;
6034
6035         PVE::Storage::activate_volumes($storecfg, [$volid]);
6036
6037         $map->{$virtdev} = $volid;
6038     }
6039
6040     return $map;
6041 };
6042
6043 sub restore_update_config_line {
6044     my ($cookie, $map, $line, $unique) = @_;
6045
6046     return '' if $line =~ m/^\#qmdump\#/;
6047     return '' if $line =~ m/^\#vzdump\#/;
6048     return '' if $line =~ m/^lock:/;
6049     return '' if $line =~ m/^unused\d+:/;
6050     return '' if $line =~ m/^parent:/;
6051
6052     my $res = '';
6053
6054     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6055     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6056         # try to convert old 1.X settings
6057         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6058         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6059             my ($model, $macaddr) = split(/\=/, $devconfig);
6060             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6061             my $net = {
6062                 model => $model,
6063                 bridge => "vmbr$ind",
6064                 macaddr => $macaddr,
6065             };
6066             my $netstr = print_net($net);
6067
6068             $res .= "net$cookie->{netcount}: $netstr\n";
6069             $cookie->{netcount}++;
6070         }
6071     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6072         my ($id, $netstr) = ($1, $2);
6073         my $net = parse_net($netstr);
6074         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6075         $netstr = print_net($net);
6076         $res .= "$id: $netstr\n";
6077     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
6078         my $virtdev = $1;
6079         my $value = $3;
6080         my $di = parse_drive($virtdev, $value);
6081         if (defined($di->{backup}) && !$di->{backup}) {
6082             $res .= "#$line";
6083         } elsif ($map->{$virtdev}) {
6084             delete $di->{format}; # format can change on restore
6085             $di->{file} = $map->{$virtdev};
6086             $value = print_drive($di);
6087             $res .= "$virtdev: $value\n";
6088         } else {
6089             $res .= $line;
6090         }
6091     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6092         my $vmgenid = $1;
6093         if ($vmgenid ne '0') {
6094             # always generate a new vmgenid if there was a valid one setup
6095             $vmgenid = generate_uuid();
6096         }
6097         $res .= "vmgenid: $vmgenid\n";
6098     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6099         my ($uuid, $uuid_str);
6100         UUID::generate($uuid);
6101         UUID::unparse($uuid, $uuid_str);
6102         my $smbios1 = parse_smbios1($2);
6103         $smbios1->{uuid} = $uuid_str;
6104         $res .= $1.print_smbios1($smbios1)."\n";
6105     } else {
6106         $res .= $line;
6107     }
6108
6109     return $res;
6110 }
6111
6112 my $restore_deactivate_volumes = sub {
6113     my ($storecfg, $devinfo) = @_;
6114
6115     my $vollist = [];
6116     foreach my $devname (keys %$devinfo) {
6117         my $volid = $devinfo->{$devname}->{volid};
6118         push @$vollist, $volid if $volid;
6119     }
6120
6121     PVE::Storage::deactivate_volumes($storecfg, $vollist);
6122 };
6123
6124 my $restore_destroy_volumes = sub {
6125     my ($storecfg, $devinfo) = @_;
6126
6127     foreach my $devname (keys %$devinfo) {
6128         my $volid = $devinfo->{$devname}->{volid};
6129         next if !$volid;
6130         eval {
6131             if ($volid =~ m|^/|) {
6132                 unlink $volid || die 'unlink failed\n';
6133             } else {
6134                 PVE::Storage::vdisk_free($storecfg, $volid);
6135             }
6136             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6137         };
6138         print STDERR "unable to cleanup '$volid' - $@" if $@;
6139     }
6140 };
6141
6142 sub scan_volids {
6143     my ($cfg, $vmid) = @_;
6144
6145     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6146
6147     my $volid_hash = {};
6148     foreach my $storeid (keys %$info) {
6149         foreach my $item (@{$info->{$storeid}}) {
6150             next if !($item->{volid} && $item->{size});
6151             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6152             $volid_hash->{$item->{volid}} = $item;
6153         }
6154     }
6155
6156     return $volid_hash;
6157 }
6158
6159 sub update_disk_config {
6160     my ($vmid, $conf, $volid_hash) = @_;
6161
6162     my $changes;
6163     my $prefix = "VM $vmid";
6164
6165     # used and unused disks
6166     my $referenced = {};
6167
6168     # Note: it is allowed to define multiple storages with same path (alias), so
6169     # we need to check both 'volid' and real 'path' (two different volid can point
6170     # to the same path).
6171
6172     my $referencedpath = {};
6173
6174     # update size info
6175     PVE::QemuConfig->foreach_volume($conf, sub {
6176         my ($opt, $drive) = @_;
6177
6178         my $volid = $drive->{file};
6179         return if !$volid;
6180         my $volume = $volid_hash->{$volid};
6181
6182         # mark volid as "in-use" for next step
6183         $referenced->{$volid} = 1;
6184         if ($volume && (my $path = $volume->{path})) {
6185             $referencedpath->{$path} = 1;
6186         }
6187
6188         return if drive_is_cdrom($drive);
6189         return if !$volume;
6190
6191         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6192         if (defined($updated)) {
6193             $changes = 1;
6194             $conf->{$opt} = print_drive($updated);
6195             print "$prefix ($opt): $msg\n";
6196         }
6197     });
6198
6199     # remove 'unusedX' entry if volume is used
6200     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6201         my ($opt, $drive) = @_;
6202
6203         my $volid = $drive->{file};
6204         return if !$volid;
6205
6206         my $path;
6207         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6208         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6209             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6210             $changes = 1;
6211             delete $conf->{$opt};
6212         }
6213
6214         $referenced->{$volid} = 1;
6215         $referencedpath->{$path} = 1 if $path;
6216     });
6217
6218     foreach my $volid (sort keys %$volid_hash) {
6219         next if $volid =~ m/vm-$vmid-state-/;
6220         next if $referenced->{$volid};
6221         my $path = $volid_hash->{$volid}->{path};
6222         next if !$path; # just to be sure
6223         next if $referencedpath->{$path};
6224         $changes = 1;
6225         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6226         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6227         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6228     }
6229
6230     return $changes;
6231 }
6232
6233 sub rescan {
6234     my ($vmid, $nolock, $dryrun) = @_;
6235
6236     my $cfg = PVE::Storage::config();
6237
6238     print "rescan volumes...\n";
6239     my $volid_hash = scan_volids($cfg, $vmid);
6240
6241     my $updatefn =  sub {
6242         my ($vmid) = @_;
6243
6244         my $conf = PVE::QemuConfig->load_config($vmid);
6245
6246         PVE::QemuConfig->check_lock($conf);
6247
6248         my $vm_volids = {};
6249         foreach my $volid (keys %$volid_hash) {
6250             my $info = $volid_hash->{$volid};
6251             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6252         }
6253
6254         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6255
6256         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6257     };
6258
6259     if (defined($vmid)) {
6260         if ($nolock) {
6261             &$updatefn($vmid);
6262         } else {
6263             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6264         }
6265     } else {
6266         my $vmlist = config_list();
6267         foreach my $vmid (keys %$vmlist) {
6268             if ($nolock) {
6269                 &$updatefn($vmid);
6270             } else {
6271                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6272             }
6273         }
6274     }
6275 }
6276
6277 sub restore_proxmox_backup_archive {
6278     my ($archive, $vmid, $user, $options) = @_;
6279
6280     my $storecfg = PVE::Storage::config();
6281
6282     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6283     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6284
6285     my $fingerprint = $scfg->{fingerprint};
6286     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6287
6288     my $repo = PVE::PBSClient::get_repository($scfg);
6289
6290     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6291     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6292     local $ENV{PBS_PASSWORD} = $password;
6293     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6294
6295     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6296         PVE::Storage::parse_volname($storecfg, $archive);
6297
6298     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6299
6300     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6301
6302     my $tmpdir = "/var/tmp/vzdumptmp$$";
6303     rmtree $tmpdir;
6304     mkpath $tmpdir;
6305
6306     my $conffile = PVE::QemuConfig->config_file($vmid);
6307      # disable interrupts (always do cleanups)
6308     local $SIG{INT} =
6309         local $SIG{TERM} =
6310         local $SIG{QUIT} =
6311         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6312
6313     # Note: $oldconf is undef if VM does not exists
6314     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6315     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6316     my $new_conf_raw = '';
6317
6318     my $rpcenv = PVE::RPCEnvironment::get();
6319     my $devinfo = {};
6320
6321     eval {
6322         # enable interrupts
6323         local $SIG{INT} =
6324             local $SIG{TERM} =
6325             local $SIG{QUIT} =
6326             local $SIG{HUP} =
6327             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6328
6329         my $cfgfn = "$tmpdir/qemu-server.conf";
6330         my $firewall_config_fn = "$tmpdir/fw.conf";
6331         my $index_fn = "$tmpdir/index.json";
6332
6333         my $cmd = "restore";
6334
6335         my $param = [$pbs_backup_name, "index.json", $index_fn];
6336         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6337         my $index = PVE::Tools::file_get_contents($index_fn);
6338         $index = decode_json($index);
6339
6340         # print Dumper($index);
6341         foreach my $info (@{$index->{files}}) {
6342             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6343                 my $devname = $1;
6344                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6345                     $devinfo->{$devname}->{size} = $1;
6346                 } else {
6347                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6348                 }
6349             }
6350         }
6351
6352         my $is_qemu_server_backup = scalar(
6353             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6354         );
6355         if (!$is_qemu_server_backup) {
6356             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6357         }
6358         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6359
6360         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6361         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6362
6363         if ($has_firewall_config) {
6364             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6365             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6366
6367             my $pve_firewall_dir = '/etc/pve/firewall';
6368             mkdir $pve_firewall_dir; # make sure the dir exists
6369             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6370         }
6371
6372         my $fh = IO::File->new($cfgfn, "r") ||
6373             die "unable to read qemu-server.conf - $!\n";
6374
6375         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6376
6377         # fixme: rate limit?
6378
6379         # create empty/temp config
6380         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6381
6382         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6383
6384         # allocate volumes
6385         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6386
6387         foreach my $virtdev (sort keys %$virtdev_hash) {
6388             my $d = $virtdev_hash->{$virtdev};
6389             next if $d->{is_cloudinit}; # no need to restore cloudinit
6390
6391             # this fails if storage is unavailable
6392             my $volid = $d->{volid};
6393             my $path = PVE::Storage::path($storecfg, $volid);
6394
6395             # for live-restore we only want to preload the efidisk
6396             next if $options->{live} && $virtdev ne 'efidisk0';
6397
6398             my $pbs_restore_cmd = [
6399                 '/usr/bin/pbs-restore',
6400                 '--repository', $repo,
6401                 $pbs_backup_name,
6402                 "$d->{devname}.img.fidx",
6403                 $path,
6404                 '--verbose',
6405                 ];
6406
6407             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6408             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
6409
6410             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6411                 push @$pbs_restore_cmd, '--skip-zero';
6412             }
6413
6414             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6415             print "restore proxmox backup image: $dbg_cmdstring\n";
6416             run_command($pbs_restore_cmd);
6417         }
6418
6419         $fh->seek(0, 0) || die "seek failed - $!\n";
6420
6421         my $cookie = { netcount => 0 };
6422         while (defined(my $line = <$fh>)) {
6423             $new_conf_raw .= restore_update_config_line(
6424                 $cookie,
6425                 $map,
6426                 $line,
6427                 $options->{unique},
6428             );
6429         }
6430
6431         $fh->close();
6432     };
6433     my $err = $@;
6434
6435     if ($err || !$options->{live}) {
6436         $restore_deactivate_volumes->($storecfg, $devinfo);
6437     }
6438
6439     rmtree $tmpdir;
6440
6441     if ($err) {
6442         $restore_destroy_volumes->($storecfg, $devinfo);
6443         die $err;
6444     }
6445
6446     if ($options->{live}) {
6447         # keep lock during live-restore
6448         $new_conf_raw .= "\nlock: create";
6449     }
6450
6451     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6452
6453     PVE::Cluster::cfs_update(); # make sure we read new file
6454
6455     eval { rescan($vmid, 1); };
6456     warn $@ if $@;
6457
6458     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
6459
6460     if ($options->{live}) {
6461         # enable interrupts
6462         local $SIG{INT} =
6463             local $SIG{TERM} =
6464             local $SIG{QUIT} =
6465             local $SIG{HUP} =
6466             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
6467
6468         my $conf = PVE::QemuConfig->load_config($vmid);
6469         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
6470
6471         delete $devinfo->{'drive-efidisk0'}; # this special drive is already restored before start
6472         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $repo, $keyfile, $pbs_backup_name);
6473
6474         PVE::QemuConfig->remove_lock($vmid, "create");
6475     }
6476 }
6477
6478 sub pbs_live_restore {
6479     my ($vmid, $conf, $storecfg, $restored_disks, $repo, $keyfile, $snap) = @_;
6480
6481     print "starting VM for live-restore\n";
6482     print "repository: '$repo', snapshot: '$snap'\n";
6483
6484     my $pbs_backing = {};
6485     for my $ds (keys %$restored_disks) {
6486         $ds =~ m/^drive-(.*)$/;
6487         my $confname = $1;
6488         $pbs_backing->{$confname} = {
6489             repository => $repo,
6490             snapshot => $snap,
6491             archive => "$ds.img.fidx",
6492         };
6493         $pbs_backing->{$confname}->{keyfile} = $keyfile if -e $keyfile;
6494
6495         my $drive = parse_drive($confname, $conf->{$confname});
6496         print "restoring '$ds' to '$drive->{file}'\n";
6497     }
6498
6499     my $drives_streamed = 0;
6500     eval {
6501         # make sure HA doesn't interrupt our restore by stopping the VM
6502         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
6503             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
6504         }
6505
6506         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
6507         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
6508         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
6509
6510         my $qmeventd_fd = register_qmeventd_handle($vmid);
6511
6512         # begin streaming, i.e. data copy from PBS to target disk for every vol,
6513         # this will effectively collapse the backing image chain consisting of
6514         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
6515         # removes itself once all backing images vanish with 'auto-remove=on')
6516         my $jobs = {};
6517         for my $ds (sort keys %$restored_disks) {
6518             my $job_id = "restore-$ds";
6519             mon_cmd($vmid, 'block-stream',
6520                 'job-id' => $job_id,
6521                 device => "$ds",
6522             );
6523             $jobs->{$job_id} = {};
6524         }
6525
6526         mon_cmd($vmid, 'cont');
6527         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
6528
6529         print "restore-drive jobs finished successfully, removing all tracking block devices"
6530             ." to disconnect from Proxmox Backup Server\n";
6531
6532         for my $ds (sort keys %$restored_disks) {
6533             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
6534         }
6535
6536         close($qmeventd_fd);
6537     };
6538
6539     my $err = $@;
6540
6541     if ($err) {
6542         warn "An error occured during live-restore: $err\n";
6543         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
6544         die "live-restore failed\n";
6545     }
6546 }
6547
6548 sub restore_vma_archive {
6549     my ($archive, $vmid, $user, $opts, $comp) = @_;
6550
6551     my $readfrom = $archive;
6552
6553     my $cfg = PVE::Storage::config();
6554     my $commands = [];
6555     my $bwlimit = $opts->{bwlimit};
6556
6557     my $dbg_cmdstring = '';
6558     my $add_pipe = sub {
6559         my ($cmd) = @_;
6560         push @$commands, $cmd;
6561         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6562         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6563         $readfrom = '-';
6564     };
6565
6566     my $input = undef;
6567     if ($archive eq '-') {
6568         $input = '<&STDIN';
6569     } else {
6570         # If we use a backup from a PVE defined storage we also consider that
6571         # storage's rate limit:
6572         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6573         if (defined($volid)) {
6574             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6575             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6576             if ($readlimit) {
6577                 print STDERR "applying read rate limit: $readlimit\n";
6578                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6579                 $add_pipe->($cstream);
6580             }
6581         }
6582     }
6583
6584     if ($comp) {
6585         my $info = PVE::Storage::decompressor_info('vma', $comp);
6586         my $cmd = $info->{decompressor};
6587         push @$cmd, $readfrom;
6588         $add_pipe->($cmd);
6589     }
6590
6591     my $tmpdir = "/var/tmp/vzdumptmp$$";
6592     rmtree $tmpdir;
6593
6594     # disable interrupts (always do cleanups)
6595     local $SIG{INT} =
6596         local $SIG{TERM} =
6597         local $SIG{QUIT} =
6598         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6599
6600     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6601     POSIX::mkfifo($mapfifo, 0600);
6602     my $fifofh;
6603     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
6604
6605     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6606
6607     my $oldtimeout;
6608     my $timeout = 5;
6609
6610     my $devinfo = {};
6611
6612     my $rpcenv = PVE::RPCEnvironment::get();
6613
6614     my $conffile = PVE::QemuConfig->config_file($vmid);
6615
6616     # Note: $oldconf is undef if VM does not exist
6617     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6618     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6619     my $new_conf_raw = '';
6620
6621     my %storage_limits;
6622
6623     my $print_devmap = sub {
6624         my $cfgfn = "$tmpdir/qemu-server.conf";
6625
6626         # we can read the config - that is already extracted
6627         my $fh = IO::File->new($cfgfn, "r") ||
6628             die "unable to read qemu-server.conf - $!\n";
6629
6630         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6631         if (-f $fwcfgfn) {
6632             my $pve_firewall_dir = '/etc/pve/firewall';
6633             mkdir $pve_firewall_dir; # make sure the dir exists
6634             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6635         }
6636
6637         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6638
6639         foreach my $info (values %{$virtdev_hash}) {
6640             my $storeid = $info->{storeid};
6641             next if defined($storage_limits{$storeid});
6642
6643             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
6644             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
6645             $storage_limits{$storeid} = $limit * 1024;
6646         }
6647
6648         foreach my $devname (keys %$devinfo) {
6649             die "found no device mapping information for device '$devname'\n"
6650                 if !$devinfo->{$devname}->{virtdev};
6651         }
6652
6653         # create empty/temp config
6654         if ($oldconf) {
6655             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6656             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6657         }
6658
6659         # allocate volumes
6660         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6661
6662         # print restore information to $fifofh
6663         foreach my $virtdev (sort keys %$virtdev_hash) {
6664             my $d = $virtdev_hash->{$virtdev};
6665             next if $d->{is_cloudinit}; # no need to restore cloudinit
6666
6667             my $storeid = $d->{storeid};
6668             my $volid = $d->{volid};
6669
6670             my $map_opts = '';
6671             if (my $limit = $storage_limits{$storeid}) {
6672                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6673             }
6674
6675             my $write_zeros = 1;
6676             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6677                 $write_zeros = 0;
6678             }
6679
6680             my $path = PVE::Storage::path($cfg, $volid);
6681
6682             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6683
6684             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6685         }
6686
6687         $fh->seek(0, 0) || die "seek failed - $!\n";
6688
6689         my $cookie = { netcount => 0 };
6690         while (defined(my $line = <$fh>)) {
6691             $new_conf_raw .= restore_update_config_line(
6692                 $cookie,
6693                 $map,
6694                 $line,
6695                 $opts->{unique},
6696             );
6697         }
6698
6699         $fh->close();
6700     };
6701
6702     eval {
6703         # enable interrupts
6704         local $SIG{INT} =
6705             local $SIG{TERM} =
6706             local $SIG{QUIT} =
6707             local $SIG{HUP} =
6708             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6709         local $SIG{ALRM} = sub { die "got timeout\n"; };
6710
6711         $oldtimeout = alarm($timeout);
6712
6713         my $parser = sub {
6714             my $line = shift;
6715
6716             print "$line\n";
6717
6718             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6719                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6720                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6721             } elsif ($line =~ m/^CTIME: /) {
6722                 # we correctly received the vma config, so we can disable
6723                 # the timeout now for disk allocation (set to 10 minutes, so
6724                 # that we always timeout if something goes wrong)
6725                 alarm(600);
6726                 &$print_devmap();
6727                 print $fifofh "done\n";
6728                 my $tmp = $oldtimeout || 0;
6729                 $oldtimeout = undef;
6730                 alarm($tmp);
6731                 close($fifofh);
6732                 $fifofh = undef;
6733             }
6734         };
6735
6736         print "restore vma archive: $dbg_cmdstring\n";
6737         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6738     };
6739     my $err = $@;
6740
6741     alarm($oldtimeout) if $oldtimeout;
6742
6743     $restore_deactivate_volumes->($cfg, $devinfo);
6744
6745     close($fifofh) if $fifofh;
6746     unlink $mapfifo;
6747     rmtree $tmpdir;
6748
6749     if ($err) {
6750         $restore_destroy_volumes->($cfg, $devinfo);
6751         die $err;
6752     }
6753
6754     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6755
6756     PVE::Cluster::cfs_update(); # make sure we read new file
6757
6758     eval { rescan($vmid, 1); };
6759     warn $@ if $@;
6760
6761     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
6762 }
6763
6764 sub restore_tar_archive {
6765     my ($archive, $vmid, $user, $opts) = @_;
6766
6767     if ($archive ne '-') {
6768         my $firstfile = tar_archive_read_firstfile($archive);
6769         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
6770             if $firstfile ne 'qemu-server.conf';
6771     }
6772
6773     my $storecfg = PVE::Storage::config();
6774
6775     # avoid zombie disks when restoring over an existing VM -> cleanup first
6776     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6777     # skiplock=1 because qmrestore has set the 'create' lock itself already
6778     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6779     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6780
6781     my $tocmd = "/usr/lib/qemu-server/qmextract";
6782
6783     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6784     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6785     $tocmd .= ' --prealloc' if $opts->{prealloc};
6786     $tocmd .= ' --info' if $opts->{info};
6787
6788     # tar option "xf" does not autodetect compression when read from STDIN,
6789     # so we pipe to zcat
6790     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6791         PVE::Tools::shellquote("--to-command=$tocmd");
6792
6793     my $tmpdir = "/var/tmp/vzdumptmp$$";
6794     mkpath $tmpdir;
6795
6796     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6797     local $ENV{VZDUMP_VMID} = $vmid;
6798     local $ENV{VZDUMP_USER} = $user;
6799
6800     my $conffile = PVE::QemuConfig->config_file($vmid);
6801     my $new_conf_raw = '';
6802
6803     # disable interrupts (always do cleanups)
6804     local $SIG{INT} =
6805         local $SIG{TERM} =
6806         local $SIG{QUIT} =
6807         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6808
6809     eval {
6810         # enable interrupts
6811         local $SIG{INT} =
6812             local $SIG{TERM} =
6813             local $SIG{QUIT} =
6814             local $SIG{HUP} =
6815             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6816
6817         if ($archive eq '-') {
6818             print "extracting archive from STDIN\n";
6819             run_command($cmd, input => "<&STDIN");
6820         } else {
6821             print "extracting archive '$archive'\n";
6822             run_command($cmd);
6823         }
6824
6825         return if $opts->{info};
6826
6827         # read new mapping
6828         my $map = {};
6829         my $statfile = "$tmpdir/qmrestore.stat";
6830         if (my $fd = IO::File->new($statfile, "r")) {
6831             while (defined (my $line = <$fd>)) {
6832                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6833                     $map->{$1} = $2 if $1;
6834                 } else {
6835                     print STDERR "unable to parse line in statfile - $line\n";
6836                 }
6837             }
6838             $fd->close();
6839         }
6840
6841         my $confsrc = "$tmpdir/qemu-server.conf";
6842
6843         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
6844
6845         my $cookie = { netcount => 0 };
6846         while (defined (my $line = <$srcfd>)) {
6847             $new_conf_raw .= restore_update_config_line(
6848                 $cookie,
6849                 $map,
6850                 $line,
6851                 $opts->{unique},
6852             );
6853         }
6854
6855         $srcfd->close();
6856     };
6857     if (my $err = $@) {
6858         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6859         die $err;
6860     }
6861
6862     rmtree $tmpdir;
6863
6864     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6865
6866     PVE::Cluster::cfs_update(); # make sure we read new file
6867
6868     eval { rescan($vmid, 1); };
6869     warn $@ if $@;
6870 };
6871
6872 sub foreach_storage_used_by_vm {
6873     my ($conf, $func) = @_;
6874
6875     my $sidhash = {};
6876
6877     PVE::QemuConfig->foreach_volume($conf, sub {
6878         my ($ds, $drive) = @_;
6879         return if drive_is_cdrom($drive);
6880
6881         my $volid = $drive->{file};
6882
6883         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6884         $sidhash->{$sid} = $sid if $sid;
6885     });
6886
6887     foreach my $sid (sort keys %$sidhash) {
6888         &$func($sid);
6889     }
6890 }
6891
6892 my $qemu_snap_storage = {
6893     rbd => 1,
6894 };
6895 sub do_snapshots_with_qemu {
6896     my ($storecfg, $volid) = @_;
6897
6898     my $storage_name = PVE::Storage::parse_volume_id($volid);
6899     my $scfg = $storecfg->{ids}->{$storage_name};
6900     die "could not find storage '$storage_name'\n" if !defined($scfg);
6901
6902     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6903         return 1;
6904     }
6905
6906     if ($volid =~ m/\.(qcow2|qed)$/){
6907         return 1;
6908     }
6909
6910     return;
6911 }
6912
6913 sub qga_check_running {
6914     my ($vmid, $nowarn) = @_;
6915
6916     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6917     if ($@) {
6918         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6919         return 0;
6920     }
6921     return 1;
6922 }
6923
6924 sub template_create {
6925     my ($vmid, $conf, $disk) = @_;
6926
6927     my $storecfg = PVE::Storage::config();
6928
6929     PVE::QemuConfig->foreach_volume($conf, sub {
6930         my ($ds, $drive) = @_;
6931
6932         return if drive_is_cdrom($drive);
6933         return if $disk && $ds ne $disk;
6934
6935         my $volid = $drive->{file};
6936         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6937
6938         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6939         $drive->{file} = $voliddst;
6940         $conf->{$ds} = print_drive($drive);
6941         PVE::QemuConfig->write_config($vmid, $conf);
6942     });
6943 }
6944
6945 sub convert_iscsi_path {
6946     my ($path) = @_;
6947
6948     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6949         my $portal = $1;
6950         my $target = $2;
6951         my $lun = $3;
6952
6953         my $initiator_name = get_initiator_name();
6954
6955         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6956                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6957     }
6958
6959     die "cannot convert iscsi path '$path', unkown format\n";
6960 }
6961
6962 sub qemu_img_convert {
6963     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6964
6965     my $storecfg = PVE::Storage::config();
6966     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6967     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6968
6969     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6970
6971     my $cachemode;
6972     my $src_path;
6973     my $src_is_iscsi = 0;
6974     my $src_format;
6975
6976     if ($src_storeid) {
6977         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6978         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6979         $src_format = qemu_img_format($src_scfg, $src_volname);
6980         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6981         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6982         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6983     } elsif (-f $src_volid) {
6984         $src_path = $src_volid;
6985         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6986             $src_format = $1;
6987         }
6988     }
6989
6990     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6991
6992     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6993     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6994     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6995     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6996
6997     my $cmd = [];
6998     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6999     push @$cmd, '-l', "snapshot.name=$snapname"
7000         if $snapname && $src_format && $src_format eq "qcow2";
7001     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
7002     push @$cmd, '-T', $cachemode if defined($cachemode);
7003
7004     if ($src_is_iscsi) {
7005         push @$cmd, '--image-opts';
7006         $src_path = convert_iscsi_path($src_path);
7007     } elsif ($src_format) {
7008         push @$cmd, '-f', $src_format;
7009     }
7010
7011     if ($dst_is_iscsi) {
7012         push @$cmd, '--target-image-opts';
7013         $dst_path = convert_iscsi_path($dst_path);
7014     } else {
7015         push @$cmd, '-O', $dst_format;
7016     }
7017
7018     push @$cmd, $src_path;
7019
7020     if (!$dst_is_iscsi && $is_zero_initialized) {
7021         push @$cmd, "zeroinit:$dst_path";
7022     } else {
7023         push @$cmd, $dst_path;
7024     }
7025
7026     my $parser = sub {
7027         my $line = shift;
7028         if($line =~ m/\((\S+)\/100\%\)/){
7029             my $percent = $1;
7030             my $transferred = int($size * $percent / 100);
7031             my $total_h = render_bytes($size, 1);
7032             my $transferred_h = render_bytes($transferred, 1);
7033
7034             print "transferred $transferred_h of $total_h ($percent%)\n";
7035         }
7036
7037     };
7038
7039     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7040     my $err = $@;
7041     die "copy failed: $err" if $err;
7042 }
7043
7044 sub qemu_img_format {
7045     my ($scfg, $volname) = @_;
7046
7047     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7048         return $1;
7049     } else {
7050         return "raw";
7051     }
7052 }
7053
7054 sub qemu_drive_mirror {
7055     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7056
7057     $jobs = {} if !$jobs;
7058
7059     my $qemu_target;
7060     my $format;
7061     $jobs->{"drive-$drive"} = {};
7062
7063     if ($dst_volid =~ /^nbd:/) {
7064         $qemu_target = $dst_volid;
7065         $format = "nbd";
7066     } else {
7067         my $storecfg = PVE::Storage::config();
7068         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7069
7070         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7071
7072         $format = qemu_img_format($dst_scfg, $dst_volname);
7073
7074         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7075
7076         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7077     }
7078
7079     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7080     $opts->{format} = $format if $format;
7081
7082     if (defined($src_bitmap)) {
7083         $opts->{sync} = 'incremental';
7084         $opts->{bitmap} = $src_bitmap;
7085         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7086     }
7087
7088     if (defined($bwlimit)) {
7089         $opts->{speed} = $bwlimit * 1024;
7090         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7091     } else {
7092         print "drive mirror is starting for drive-$drive\n";
7093     }
7094
7095     # if a job already runs for this device we get an error, catch it for cleanup
7096     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7097     if (my $err = $@) {
7098         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7099         warn "$@\n" if $@;
7100         die "mirroring error: $err\n";
7101     }
7102
7103     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7104 }
7105
7106 # $completion can be either
7107 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7108 # 'cancel': wait until all jobs are ready, block-job-cancel them
7109 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7110 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7111 sub qemu_drive_mirror_monitor {
7112     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7113
7114     $completion //= 'complete';
7115     $op //= "mirror";
7116
7117     eval {
7118         my $err_complete = 0;
7119
7120         my $starttime = time ();
7121         while (1) {
7122             die "block job ('$op') timed out\n" if $err_complete > 300;
7123
7124             my $stats = mon_cmd($vmid, "query-block-jobs");
7125             my $ctime = time();
7126
7127             my $running_jobs = {};
7128             for my $stat (@$stats) {
7129                 next if $stat->{type} ne $op;
7130                 $running_jobs->{$stat->{device}} = $stat;
7131             }
7132
7133             my $readycounter = 0;
7134
7135             for my $job_id (sort keys %$jobs) {
7136                 my $job = $running_jobs->{$job_id};
7137
7138                 my $vanished = !defined($job);
7139                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7140                 if($complete || ($vanished && $completion eq 'auto')) {
7141                     print "$job_id: $op-job finished\n";
7142                     delete $jobs->{$job_id};
7143                     next;
7144                 }
7145
7146                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7147
7148                 my $busy = $job->{busy};
7149                 my $ready = $job->{ready};
7150                 if (my $total = $job->{len}) {
7151                     my $transferred = $job->{offset} || 0;
7152                     my $remaining = $total - $transferred;
7153                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7154
7155                     my $duration = $ctime - $starttime;
7156                     my $total_h = render_bytes($total, 1);
7157                     my $transferred_h = render_bytes($transferred, 1);
7158
7159                     my $status = sprintf(
7160                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7161                         render_duration($duration),
7162                     );
7163
7164                     if ($ready) {
7165                         if ($busy) {
7166                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7167                         } else {
7168                             $status .= ", ready";
7169                         }
7170                     }
7171                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7172                     $jobs->{$job_id}->{ready} = $ready;
7173                 }
7174
7175                 $readycounter++ if $job->{ready};
7176             }
7177
7178             last if scalar(keys %$jobs) == 0;
7179
7180             if ($readycounter == scalar(keys %$jobs)) {
7181                 print "all '$op' jobs are ready\n";
7182
7183                 # do the complete later (or has already been done)
7184                 last if $completion eq 'skip' || $completion eq 'auto';
7185
7186                 if ($vmiddst && $vmiddst != $vmid) {
7187                     my $agent_running = $qga && qga_check_running($vmid);
7188                     if ($agent_running) {
7189                         print "freeze filesystem\n";
7190                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7191                     } else {
7192                         print "suspend vm\n";
7193                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7194                     }
7195
7196                     # if we clone a disk for a new target vm, we don't switch the disk
7197                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7198
7199                     if ($agent_running) {
7200                         print "unfreeze filesystem\n";
7201                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7202                     } else {
7203                         print "resume vm\n";
7204                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
7205                     }
7206
7207                     last;
7208                 } else {
7209
7210                     for my $job_id (sort keys %$jobs) {
7211                         # try to switch the disk if source and destination are on the same guest
7212                         print "$job_id: Completing block job_id...\n";
7213
7214                         my $op;
7215                         if ($completion eq 'complete') {
7216                             $op = 'block-job-complete';
7217                         } elsif ($completion eq 'cancel') {
7218                             $op = 'block-job-cancel';
7219                         } else {
7220                             die "invalid completion value: $completion\n";
7221                         }
7222                         eval { mon_cmd($vmid, $op, device => $job_id) };
7223                         if ($@ =~ m/cannot be completed/) {
7224                             print "$job_id: block job cannot be completed, trying again.\n";
7225                             $err_complete++;
7226                         }else {
7227                             print "$job_id: Completed successfully.\n";
7228                             $jobs->{$job_id}->{complete} = 1;
7229                         }
7230                     }
7231                 }
7232             }
7233             sleep 1;
7234         }
7235     };
7236     my $err = $@;
7237
7238     if ($err) {
7239         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7240         die "block job ($op) error: $err";
7241     }
7242 }
7243
7244 sub qemu_blockjobs_cancel {
7245     my ($vmid, $jobs) = @_;
7246
7247     foreach my $job (keys %$jobs) {
7248         print "$job: Cancelling block job\n";
7249         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7250         $jobs->{$job}->{cancel} = 1;
7251     }
7252
7253     while (1) {
7254         my $stats = mon_cmd($vmid, "query-block-jobs");
7255
7256         my $running_jobs = {};
7257         foreach my $stat (@$stats) {
7258             $running_jobs->{$stat->{device}} = $stat;
7259         }
7260
7261         foreach my $job (keys %$jobs) {
7262
7263             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7264                 print "$job: Done.\n";
7265                 delete $jobs->{$job};
7266             }
7267         }
7268
7269         last if scalar(keys %$jobs) == 0;
7270
7271         sleep 1;
7272     }
7273 }
7274
7275 sub clone_disk {
7276     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7277         $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
7278
7279     my $newvolid;
7280
7281     if (!$full) {
7282         print "create linked clone of drive $drivename ($drive->{file})\n";
7283         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7284         push @$newvollist, $newvolid;
7285     } else {
7286
7287         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7288         $storeid = $storage if $storage;
7289
7290         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7291
7292         print "create full clone of drive $drivename ($drive->{file})\n";
7293         my $name = undef;
7294         my $size = undef;
7295         if (drive_is_cloudinit($drive)) {
7296             $name = "vm-$newvmid-cloudinit";
7297             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7298             if ($scfg->{path}) {
7299                 $name .= ".$dst_format";
7300             }
7301             $snapname = undef;
7302             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7303         } elsif ($drivename eq 'efidisk0') {
7304             $size = get_efivars_size($conf);
7305         } else {
7306             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
7307         }
7308         $newvolid = PVE::Storage::vdisk_alloc(
7309             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
7310         );
7311         push @$newvollist, $newvolid;
7312
7313         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7314
7315         if (drive_is_cloudinit($drive)) {
7316             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
7317             # if this is the case, we have to complete any block-jobs still there from
7318             # previous drive-mirrors
7319             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
7320                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
7321             }
7322             goto no_data_clone;
7323         }
7324
7325         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7326         if (!$running || $snapname) {
7327             # TODO: handle bwlimits
7328             if ($drivename eq 'efidisk0') {
7329                 # the relevant data on the efidisk may be smaller than the source
7330                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
7331                 # that is given by the OVMF_VARS.fd
7332                 my $src_path = PVE::Storage::path($storecfg, $drive->{file});
7333                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
7334
7335                 # better for Ceph if block size is not too small, see bug #3324
7336                 my $bs = 1024*1024;
7337
7338                 run_command(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=$bs", "osize=$size",
7339                     "if=$src_path", "of=$dst_path"]);
7340             } else {
7341                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7342             }
7343         } else {
7344
7345             my $kvmver = get_running_qemu_version ($vmid);
7346             if (!min_version($kvmver, 2, 7)) {
7347                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7348                     if $drive->{iothread};
7349             }
7350
7351             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7352                 $completion, $qga, $bwlimit);
7353         }
7354     }
7355
7356 no_data_clone:
7357     my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
7358
7359     my $disk = $drive;
7360     $disk->{format} = undef;
7361     $disk->{file} = $newvolid;
7362     $disk->{size} = $size if defined($size);
7363
7364     return $disk;
7365 }
7366
7367 sub get_running_qemu_version {
7368     my ($vmid) = @_;
7369     my $res = mon_cmd($vmid, "query-version");
7370     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7371 }
7372
7373 sub qemu_use_old_bios_files {
7374     my ($machine_type) = @_;
7375
7376     return if !$machine_type;
7377
7378     my $use_old_bios_files = undef;
7379
7380     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7381         $machine_type = $1;
7382         $use_old_bios_files = 1;
7383     } else {
7384         my $version = extract_version($machine_type, kvm_user_version());
7385         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7386         # load new efi bios files on migration. So this hack is required to allow
7387         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7388         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7389         $use_old_bios_files = !min_version($version, 2, 4);
7390     }
7391
7392     return ($use_old_bios_files, $machine_type);
7393 }
7394
7395 sub get_efivars_size {
7396     my ($conf) = @_;
7397     my $arch = get_vm_arch($conf);
7398     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7399     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
7400     return -s $ovmf_vars;
7401 }
7402
7403 sub update_efidisk_size {
7404     my ($conf) = @_;
7405
7406     return if !defined($conf->{efidisk0});
7407
7408     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
7409     $disk->{size} = get_efivars_size($conf);
7410     $conf->{efidisk0} = print_drive($disk);
7411
7412     return;
7413 }
7414
7415 sub create_efidisk($$$$$) {
7416     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7417
7418     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7419     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7420
7421     my $vars_size_b = -s $ovmf_vars;
7422     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7423     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7424     PVE::Storage::activate_volumes($storecfg, [$volid]);
7425
7426     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7427     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7428
7429     return ($volid, $size/1024);
7430 }
7431
7432 sub vm_iothreads_list {
7433     my ($vmid) = @_;
7434
7435     my $res = mon_cmd($vmid, 'query-iothreads');
7436
7437     my $iothreads = {};
7438     foreach my $iothread (@$res) {
7439         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7440     }
7441
7442     return $iothreads;
7443 }
7444
7445 sub scsihw_infos {
7446     my ($conf, $drive) = @_;
7447
7448     my $maxdev = 0;
7449
7450     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7451         $maxdev = 7;
7452     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7453         $maxdev = 1;
7454     } else {
7455         $maxdev = 256;
7456     }
7457
7458     my $controller = int($drive->{index} / $maxdev);
7459     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
7460         ? "virtioscsi"
7461         : "scsihw";
7462
7463     return ($maxdev, $controller, $controller_prefix);
7464 }
7465
7466 sub windows_version {
7467     my ($ostype) = @_;
7468
7469     return 0 if !$ostype;
7470
7471     my $winversion = 0;
7472
7473     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7474         $winversion = 5;
7475     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7476         $winversion = 6;
7477     } elsif ($ostype =~ m/^win(\d+)$/) {
7478         $winversion = $1;
7479     }
7480
7481     return $winversion;
7482 }
7483
7484 sub resolve_dst_disk_format {
7485         my ($storecfg, $storeid, $src_volname, $format) = @_;
7486         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7487
7488         if (!$format) {
7489             # if no target format is specified, use the source disk format as hint
7490             if ($src_volname) {
7491                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7492                 $format = qemu_img_format($scfg, $src_volname);
7493             } else {
7494                 return $defFormat;
7495             }
7496         }
7497
7498         # test if requested format is supported - else use default
7499         my $supported = grep { $_ eq $format } @$validFormats;
7500         $format = $defFormat if !$supported;
7501         return $format;
7502 }
7503
7504 # NOTE: if this logic changes, please update docs & possibly gui logic
7505 sub find_vmstate_storage {
7506     my ($conf, $storecfg) = @_;
7507
7508     # first, return storage from conf if set
7509     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7510
7511     my ($target, $shared, $local);
7512
7513     foreach_storage_used_by_vm($conf, sub {
7514         my ($sid) = @_;
7515         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7516         my $dst = $scfg->{shared} ? \$shared : \$local;
7517         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7518     });
7519
7520     # second, use shared storage where VM has at least one disk
7521     # third, use local storage where VM has at least one disk
7522     # fall back to local storage
7523     $target = $shared // $local // 'local';
7524
7525     return $target;
7526 }
7527
7528 sub generate_uuid {
7529     my ($uuid, $uuid_str);
7530     UUID::generate($uuid);
7531     UUID::unparse($uuid, $uuid_str);
7532     return $uuid_str;
7533 }
7534
7535 sub generate_smbios1_uuid {
7536     return "uuid=".generate_uuid();
7537 }
7538
7539 sub nbd_stop {
7540     my ($vmid) = @_;
7541
7542     mon_cmd($vmid, 'nbd-server-stop');
7543 }
7544
7545 sub create_reboot_request {
7546     my ($vmid) = @_;
7547     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7548         or die "failed to create reboot trigger file: $!\n";
7549     close($fh);
7550 }
7551
7552 sub clear_reboot_request {
7553     my ($vmid) = @_;
7554     my $path = "/run/qemu-server/$vmid.reboot";
7555     my $res = 0;
7556
7557     $res = unlink($path);
7558     die "could not remove reboot request for $vmid: $!"
7559         if !$res && $! != POSIX::ENOENT;
7560
7561     return $res;
7562 }
7563
7564 sub bootorder_from_legacy {
7565     my ($conf, $bootcfg) = @_;
7566
7567     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
7568     my $bootindex_hash = {};
7569     my $i = 1;
7570     foreach my $o (split(//, $boot)) {
7571         $bootindex_hash->{$o} = $i*100;
7572         $i++;
7573     }
7574
7575     my $bootorder = {};
7576
7577     PVE::QemuConfig->foreach_volume($conf, sub {
7578         my ($ds, $drive) = @_;
7579
7580         if (drive_is_cdrom ($drive, 1)) {
7581             if ($bootindex_hash->{d}) {
7582                 $bootorder->{$ds} = $bootindex_hash->{d};
7583                 $bootindex_hash->{d} += 1;
7584             }
7585         } elsif ($bootindex_hash->{c}) {
7586             $bootorder->{$ds} = $bootindex_hash->{c}
7587                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
7588             $bootindex_hash->{c} += 1;
7589         }
7590     });
7591
7592     if ($bootindex_hash->{n}) {
7593         for (my $i = 0; $i < $MAX_NETS; $i++) {
7594             my $netname = "net$i";
7595             next if !$conf->{$netname};
7596             $bootorder->{$netname} = $bootindex_hash->{n};
7597             $bootindex_hash->{n} += 1;
7598         }
7599     }
7600
7601     return $bootorder;
7602 }
7603
7604 # Generate default device list for 'boot: order=' property. Matches legacy
7605 # default boot order, but with explicit device names. This is important, since
7606 # the fallback for when neither 'order' nor the old format is specified relies
7607 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
7608 sub get_default_bootdevices {
7609     my ($conf) = @_;
7610
7611     my @ret = ();
7612
7613     # harddisk
7614     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
7615     push @ret, $first if $first;
7616
7617     # cdrom
7618     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
7619     push @ret, $first if $first;
7620
7621     # network
7622     for (my $i = 0; $i < $MAX_NETS; $i++) {
7623         my $netname = "net$i";
7624         next if !$conf->{$netname};
7625         push @ret, $netname;
7626         last;
7627     }
7628
7629     return \@ret;
7630 }
7631
7632 sub device_bootorder {
7633     my ($conf) = @_;
7634
7635     return bootorder_from_legacy($conf) if !defined($conf->{boot});
7636
7637     my $boot = parse_property_string($boot_fmt, $conf->{boot});
7638
7639     my $bootorder = {};
7640     if (!defined($boot) || $boot->{legacy}) {
7641         $bootorder = bootorder_from_legacy($conf, $boot);
7642     } elsif ($boot->{order}) {
7643         my $i = 100; # start at 100 to allow user to insert devices before us with -args
7644         for my $dev (PVE::Tools::split_list($boot->{order})) {
7645             $bootorder->{$dev} = $i++;
7646         }
7647     }
7648
7649     return $bootorder;
7650 }
7651
7652 sub register_qmeventd_handle {
7653     my ($vmid) = @_;
7654
7655     my $fh;
7656     my $peer = "/var/run/qmeventd.sock";
7657     my $count = 0;
7658
7659     for (;;) {
7660         $count++;
7661         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
7662         last if $fh;
7663         if ($! != EINTR && $! != EAGAIN) {
7664             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
7665         }
7666         if ($count > 4) {
7667             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
7668               . "after $count retries\n";
7669         }
7670         usleep(25000);
7671     }
7672
7673     # send handshake to mark VM as backing up
7674     print $fh to_json({vzdump => {vmid => "$vmid"}});
7675
7676     # return handle to be closed later when inhibit is no longer required
7677     return $fh;
7678 }
7679
7680 # bash completion helper
7681
7682 sub complete_backup_archives {
7683     my ($cmdname, $pname, $cvalue) = @_;
7684
7685     my $cfg = PVE::Storage::config();
7686
7687     my $storeid;
7688
7689     if ($cvalue =~ m/^([^:]+):/) {
7690         $storeid = $1;
7691     }
7692
7693     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7694
7695     my $res = [];
7696     foreach my $id (keys %$data) {
7697         foreach my $item (@{$data->{$id}}) {
7698             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
7699             push @$res, $item->{volid} if defined($item->{volid});
7700         }
7701     }
7702
7703     return $res;
7704 }
7705
7706 my $complete_vmid_full = sub {
7707     my ($running) = @_;
7708
7709     my $idlist = vmstatus();
7710
7711     my $res = [];
7712
7713     foreach my $id (keys %$idlist) {
7714         my $d = $idlist->{$id};
7715         if (defined($running)) {
7716             next if $d->{template};
7717             next if $running && $d->{status} ne 'running';
7718             next if !$running && $d->{status} eq 'running';
7719         }
7720         push @$res, $id;
7721
7722     }
7723     return $res;
7724 };
7725
7726 sub complete_vmid {
7727     return &$complete_vmid_full();
7728 }
7729
7730 sub complete_vmid_stopped {
7731     return &$complete_vmid_full(0);
7732 }
7733
7734 sub complete_vmid_running {
7735     return &$complete_vmid_full(1);
7736 }
7737
7738 sub complete_storage {
7739
7740     my $cfg = PVE::Storage::config();
7741     my $ids = $cfg->{ids};
7742
7743     my $res = [];
7744     foreach my $sid (keys %$ids) {
7745         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7746         next if !$ids->{$sid}->{content}->{images};
7747         push @$res, $sid;
7748     }
7749
7750     return $res;
7751 }
7752
7753 sub complete_migration_storage {
7754     my ($cmd, $param, $current_value, $all_args) = @_;
7755
7756     my $targetnode = @$all_args[1];
7757
7758     my $cfg = PVE::Storage::config();
7759     my $ids = $cfg->{ids};
7760
7761     my $res = [];
7762     foreach my $sid (keys %$ids) {
7763         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7764         next if !$ids->{$sid}->{content}->{images};
7765         push @$res, $sid;
7766     }
7767
7768     return $res;
7769 }
7770
7771 sub vm_is_paused {
7772     my ($vmid) = @_;
7773     my $qmpstatus = eval {
7774         PVE::QemuConfig::assert_config_exists_on_node($vmid);
7775         mon_cmd($vmid, "query-status");
7776     };
7777     warn "$@\n" if $@;
7778     return $qmpstatus && $qmpstatus->{status} eq "paused";
7779 }
7780
7781 sub check_volume_storage_type {
7782     my ($storecfg, $vol) = @_;
7783
7784     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
7785     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7786     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
7787
7788     die "storage '$storeid' does not support content-type '$vtype'\n"
7789         if !$scfg->{content}->{$vtype};
7790
7791     return 1;
7792 }
7793
7794 1;