PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::DataCenterConfig;
  31 use PVE::Exception qw(raise raise_param_exc);
  32 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  33 use PVE::INotify;
  34 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  35 use PVE::ProcFSTools;
  36 use PVE::RPCEnvironment;
  37 use PVE::Storage;
  38 use PVE::SysFSTools;
  39 use PVE::Systemd;
  40 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  41
  42 use PVE::QMPClient;
  43 use PVE::QemuConfig;
  44 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  45 use PVE::QemuServer::Cloudinit;
  46 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  47 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom parse_drive print_drive);
  48 use PVE::QemuServer::Machine;
  49 use PVE::QemuServer::Memory;
  50 use PVE::QemuServer::Monitor qw(mon_cmd);
  51 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  52 use PVE::QemuServer::USB qw(parse_usb_device);
  53
  54 my $have_sdn;
  55 eval {
  56     require PVE::Network::SDN::Zones;
  57     $have_sdn = 1;
  58 };
  59
  60 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  61 my $OVMF = {
  62     x86_64 => [
  63         "$EDK2_FW_BASE/OVMF_CODE.fd",
  64         "$EDK2_FW_BASE/OVMF_VARS.fd"
  65     ],
  66     aarch64 => [
  67         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  68         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  69     ],
  70 };
  71
  72 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  73
  74 # Note about locking: we use flock on the config file protect
  75 # against concurent actions.
  76 # Aditionaly, we have a 'lock' setting in the config file. This
  77 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  78 # allowed when such lock is set. But you can ignore this kind of
  79 # lock with the --skiplock flag.
  80
  81 cfs_register_file('/qemu-server/',
  82                   \&parse_vm_config,
  83                   \&write_vm_config);
  84
  85 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  86     description => "Some command save/restore state from this location.",
  87     type => 'string',
  88     maxLength => 128,
  89     optional => 1,
  90 });
  91
  92 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  93         description => "Specifies the Qemu machine type.",
  94         type => 'string',
  95         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
  96         maxLength => 40,
  97         optional => 1,
  98 });
  99
 100
 101 sub map_storage {
 102     my ($map, $source) = @_;
 103
 104     return $source if !defined($map);
 105
 106     return $map->{entries}->{$source}
 107         if $map->{entries} && defined($map->{entries}->{$source});
 108
 109     return $map->{default} if $map->{default};
 110
 111     # identity (fallback)
 112     return $source;
 113 }
 114
 115 PVE::JSONSchema::register_standard_option('pve-targetstorage', {
 116     description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
 117     type => 'string',
 118     format => 'storagepair-list',
 119     optional => 1,
 120 });
 121
 122 #no warnings 'redefine';
 123
 124 sub cgroups_write {
 125    my ($controller, $vmid, $option, $value) = @_;
 126
 127    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 128    PVE::ProcFSTools::write_proc_entry($path, $value);
 129
 130 }
 131
 132 my $nodename_cache;
 133 sub nodename {
 134     $nodename_cache //= PVE::INotify::nodename();
 135     return $nodename_cache;
 136 }
 137
 138 my $watchdog_fmt = {
 139     model => {
 140         default_key => 1,
 141         type => 'string',
 142         enum => [qw(i6300esb ib700)],
 143         description => "Watchdog type to emulate.",
 144         default => 'i6300esb',
 145         optional => 1,
 146     },
 147     action => {
 148         type => 'string',
 149         enum => [qw(reset shutdown poweroff pause debug none)],
 150         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 151         optional => 1,
 152     },
 153 };
 154 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 155
 156 my $agent_fmt = {
 157     enabled => {
 158         description => "Enable/disable Qemu GuestAgent.",
 159         type => 'boolean',
 160         default => 0,
 161         default_key => 1,
 162     },
 163     fstrim_cloned_disks => {
 164         description => "Run fstrim after cloning/moving a disk.",
 165         type => 'boolean',
 166         optional => 1,
 167         default => 0
 168     },
 169     type => {
 170         description => "Select the agent type",
 171         type => 'string',
 172         default => 'virtio',
 173         optional => 1,
 174         enum => [qw(virtio isa)],
 175     },
 176 };
 177
 178 my $vga_fmt = {
 179     type => {
 180         description => "Select the VGA type.",
 181         type => 'string',
 182         default => 'std',
 183         optional => 1,
 184         default_key => 1,
 185         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 186     },
 187     memory => {
 188         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 189         type => 'integer',
 190         optional => 1,
 191         minimum => 4,
 192         maximum => 512,
 193     },
 194 };
 195
 196 my $ivshmem_fmt = {
 197     size => {
 198         type => 'integer',
 199         minimum => 1,
 200         description => "The size of the file in MB.",
 201     },
 202     name => {
 203         type => 'string',
 204         pattern => '[a-zA-Z0-9\-]+',
 205         optional => 1,
 206         format_description => 'string',
 207         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 208     },
 209 };
 210
 211 my $audio_fmt = {
 212     device => {
 213         type => 'string',
 214         enum => [qw(ich9-intel-hda intel-hda AC97)],
 215         description =>  "Configure an audio device."
 216     },
 217     driver =>  {
 218         type => 'string',
 219         enum => ['spice'],
 220         default => 'spice',
 221         optional => 1,
 222         description => "Driver backend for the audio device."
 223     },
 224 };
 225
 226 my $spice_enhancements_fmt = {
 227     foldersharing => {
 228         type => 'boolean',
 229         optional => 1,
 230         default => '0',
 231         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 232     },
 233     videostreaming =>  {
 234         type => 'string',
 235         enum => ['off', 'all', 'filter'],
 236         default => 'off',
 237         optional => 1,
 238         description => "Enable video streaming. Uses compression for detected video streams."
 239     },
 240 };
 241
 242 my $rng_fmt = {
 243     source => {
 244         type => 'string',
 245         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 246         default_key => 1,
 247         description => "The file on the host to gather entropy from. In most"
 248                      . " cases /dev/urandom should be preferred over /dev/random"
 249                      . " to avoid entropy-starvation issues on the host. Using"
 250                      . " urandom does *not* decrease security in any meaningful"
 251                      . " way, as it's still seeded from real entropy, and the"
 252                      . " bytes provided will most likely be mixed with real"
 253                      . " entropy on the guest as well. /dev/hwrng can be used"
 254                      . " to pass through a hardware RNG from the host.",
 255     },
 256     max_bytes => {
 257         type => 'integer',
 258         description => "Maximum bytes of entropy injected into the guest every"
 259                      . " 'period' milliseconds. Prefer a lower value when using"
 260                      . " /dev/random as source. Use 0 to disable limiting"
 261                      . " (potentially dangerous!).",
 262         optional => 1,
 263
 264         # default is 1 KiB/s, provides enough entropy to the guest to avoid
 265         # boot-starvation issues (e.g. systemd etc...) while allowing no chance
 266         # of overwhelming the host, provided we're reading from /dev/urandom
 267         default => 1024,
 268     },
 269     period => {
 270         type => 'integer',
 271         description => "Every 'period' milliseconds the entropy-injection quota"
 272                      . " is reset, allowing the guest to retrieve another"
 273                      . " 'max_bytes' of entropy.",
 274         optional => 1,
 275         default => 1000,
 276     },
 277 };
 278
 279 my $confdesc = {
 280     onboot => {
 281         optional => 1,
 282         type => 'boolean',
 283         description => "Specifies whether a VM will be started during system bootup.",
 284         default => 0,
 285     },
 286     autostart => {
 287         optional => 1,
 288         type => 'boolean',
 289         description => "Automatic restart after crash (currently ignored).",
 290         default => 0,
 291     },
 292     hotplug => {
 293         optional => 1,
 294         type => 'string', format => 'pve-hotplug-features',
 295         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 296         default => 'network,disk,usb',
 297     },
 298     reboot => {
 299         optional => 1,
 300         type => 'boolean',
 301         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 302         default => 1,
 303     },
 304     lock => {
 305         optional => 1,
 306         type => 'string',
 307         description => "Lock/unlock the VM.",
 308         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 309     },
 310     cpulimit => {
 311         optional => 1,
 312         type => 'number',
 313         description => "Limit of CPU usage.",
 314         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 315         minimum => 0,
 316         maximum => 128,
 317         default => 0,
 318     },
 319     cpuunits => {
 320         optional => 1,
 321         type => 'integer',
 322         description => "CPU weight for a VM.",
 323         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 324         minimum => 2,
 325         maximum => 262144,
 326         default => 1024,
 327     },
 328     memory => {
 329         optional => 1,
 330         type => 'integer',
 331         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 332         minimum => 16,
 333         default => 512,
 334     },
 335     balloon => {
 336         optional => 1,
 337         type => 'integer',
 338         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 339         minimum => 0,
 340     },
 341     shares => {
 342         optional => 1,
 343         type => 'integer',
 344         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 345         minimum => 0,
 346         maximum => 50000,
 347         default => 1000,
 348     },
 349     keyboard => {
 350         optional => 1,
 351         type => 'string',
 352         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 353                        "It should not be necessary to set it.",
 354         enum => PVE::Tools::kvmkeymaplist(),
 355         default => undef,
 356     },
 357     name => {
 358         optional => 1,
 359         type => 'string', format => 'dns-name',
 360         description => "Set a name for the VM. Only used on the configuration web interface.",
 361     },
 362     scsihw => {
 363         optional => 1,
 364         type => 'string',
 365         description => "SCSI controller model",
 366         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 367         default => 'lsi',
 368     },
 369     description => {
 370         optional => 1,
 371         type => 'string',
 372         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 373     },
 374     ostype => {
 375         optional => 1,
 376         type => 'string',
 377         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 378         description => "Specify guest operating system.",
 379         verbose_description => <<EODESC,
 380 Specify guest operating system. This is used to enable special
 381 optimization/features for specific operating systems:
 382
 383 [horizontal]
 384 other;; unspecified OS
 385 wxp;; Microsoft Windows XP
 386 w2k;; Microsoft Windows 2000
 387 w2k3;; Microsoft Windows 2003
 388 w2k8;; Microsoft Windows 2008
 389 wvista;; Microsoft Windows Vista
 390 win7;; Microsoft Windows 7
 391 win8;; Microsoft Windows 8/2012/2012r2
 392 win10;; Microsoft Windows 10/2016
 393 l24;; Linux 2.4 Kernel
 394 l26;; Linux 2.6 - 5.X Kernel
 395 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 396 EODESC
 397     },
 398     boot => {
 399         optional => 1,
 400         type => 'string', format => 'pve-qm-boot',
 401         description => "Specify guest boot order. Use with 'order=', usage with"
 402                      . " no key or 'legacy=' is deprecated.",
 403     },
 404     bootdisk => {
 405         optional => 1,
 406         type => 'string', format => 'pve-qm-bootdisk',
 407         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 408         pattern => '(ide|sata|scsi|virtio)\d+',
 409     },
 410     smp => {
 411         optional => 1,
 412         type => 'integer',
 413         description => "The number of CPUs. Please use option -sockets instead.",
 414         minimum => 1,
 415         default => 1,
 416     },
 417     sockets => {
 418         optional => 1,
 419         type => 'integer',
 420         description => "The number of CPU sockets.",
 421         minimum => 1,
 422         default => 1,
 423     },
 424     cores => {
 425         optional => 1,
 426         type => 'integer',
 427         description => "The number of cores per socket.",
 428         minimum => 1,
 429         default => 1,
 430     },
 431     numa => {
 432         optional => 1,
 433         type => 'boolean',
 434         description => "Enable/disable NUMA.",
 435         default => 0,
 436     },
 437     hugepages => {
 438         optional => 1,
 439         type => 'string',
 440         description => "Enable/disable hugepages memory.",
 441         enum => [qw(any 2 1024)],
 442     },
 443     keephugepages => {
 444         optional => 1,
 445         type => 'boolean',
 446         default => 0,
 447         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 448             ." after VM shutdown and can be used for subsequent starts.",
 449     },
 450     vcpus => {
 451         optional => 1,
 452         type => 'integer',
 453         description => "Number of hotplugged vcpus.",
 454         minimum => 1,
 455         default => 0,
 456     },
 457     acpi => {
 458         optional => 1,
 459         type => 'boolean',
 460         description => "Enable/disable ACPI.",
 461         default => 1,
 462     },
 463     agent => {
 464         optional => 1,
 465         description => "Enable/disable Qemu GuestAgent and its properties.",
 466         type => 'string',
 467         format => $agent_fmt,
 468     },
 469     kvm => {
 470         optional => 1,
 471         type => 'boolean',
 472         description => "Enable/disable KVM hardware virtualization.",
 473         default => 1,
 474     },
 475     tdf => {
 476         optional => 1,
 477         type => 'boolean',
 478         description => "Enable/disable time drift fix.",
 479         default => 0,
 480     },
 481     localtime => {
 482         optional => 1,
 483         type => 'boolean',
 484         description => "Set the real time clock to local time. This is enabled by default if ostype"
 485             ." indicates a Microsoft OS.",
 486     },
 487     freeze => {
 488         optional => 1,
 489         type => 'boolean',
 490         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 491     },
 492     vga => {
 493         optional => 1,
 494         type => 'string', format => $vga_fmt,
 495         description => "Configure the VGA hardware.",
 496         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 497             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 498             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 499             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 500             ." display server. For win* OS you can select how many independent displays you want,"
 501             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 502             ." using a serial device as terminal.",
 503     },
 504     watchdog => {
 505         optional => 1,
 506         type => 'string', format => 'pve-qm-watchdog',
 507         description => "Create a virtual hardware watchdog device.",
 508         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 509             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 510             ." else the watchdog will reset the guest (or execute the respective action specified)",
 511     },
 512     startdate => {
 513         optional => 1,
 514         type => 'string',
 515         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 516         description => "Set the initial date of the real time clock. Valid format for date are:"
 517             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 518         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 519         default => 'now',
 520     },
 521     startup =>  get_standard_option('pve-startup-order'),
 522     template => {
 523         optional => 1,
 524         type => 'boolean',
 525         description => "Enable/disable Template.",
 526         default => 0,
 527     },
 528     args => {
 529         optional => 1,
 530         type => 'string',
 531         description => "Arbitrary arguments passed to kvm.",
 532         verbose_description => <<EODESCR,
 533 Arbitrary arguments passed to kvm, for example:
 534
 535 args: -no-reboot -no-hpet
 536
 537 NOTE: this option is for experts only.
 538 EODESCR
 539     },
 540     tablet => {
 541         optional => 1,
 542         type => 'boolean',
 543         default => 1,
 544         description => "Enable/disable the USB tablet device.",
 545         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 546             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 547             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 548             ." may consider disabling this to save some context switches. This is turned off by"
 549             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 550     },
 551     migrate_speed => {
 552         optional => 1,
 553         type => 'integer',
 554         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 555         minimum => 0,
 556         default => 0,
 557     },
 558     migrate_downtime => {
 559         optional => 1,
 560         type => 'number',
 561         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 562         minimum => 0,
 563         default => 0.1,
 564     },
 565     cdrom => {
 566         optional => 1,
 567         type => 'string', format => 'pve-qm-ide',
 568         typetext => '<volume>',
 569         description => "This is an alias for option -ide2",
 570     },
 571     cpu => {
 572         optional => 1,
 573         description => "Emulated CPU type.",
 574         type => 'string',
 575         format => 'pve-vm-cpu-conf',
 576     },
 577     parent => get_standard_option('pve-snapshot-name', {
 578         optional => 1,
 579         description => "Parent snapshot name. This is used internally, and should not be modified.",
 580     }),
 581     snaptime => {
 582         optional => 1,
 583         description => "Timestamp for snapshots.",
 584         type => 'integer',
 585         minimum => 0,
 586     },
 587     vmstate => {
 588         optional => 1,
 589         type => 'string', format => 'pve-volume-id',
 590         description => "Reference to a volume which stores the VM state. This is used internally"
 591             ." for snapshots.",
 592     },
 593     vmstatestorage => get_standard_option('pve-storage-id', {
 594         description => "Default storage for VM state volumes/files.",
 595         optional => 1,
 596     }),
 597     runningmachine => get_standard_option('pve-qemu-machine', {
 598         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 599             ." for snapshots.",
 600     }),
 601     runningcpu => {
 602         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 603             ." internally for snapshots.",
 604         optional => 1,
 605         type => 'string',
 606         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 607         format_description => 'QEMU -cpu parameter'
 608     },
 609     machine => get_standard_option('pve-qemu-machine'),
 610     arch => {
 611         description => "Virtual processor architecture. Defaults to the host.",
 612         optional => 1,
 613         type => 'string',
 614         enum => [qw(x86_64 aarch64)],
 615     },
 616     smbios1 => {
 617         description => "Specify SMBIOS type 1 fields.",
 618         type => 'string', format => 'pve-qm-smbios1',
 619         maxLength => 512,
 620         optional => 1,
 621     },
 622     protection => {
 623         optional => 1,
 624         type => 'boolean',
 625         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 626             ." remove disk operations.",
 627         default => 0,
 628     },
 629     bios => {
 630         optional => 1,
 631         type => 'string',
 632         enum => [ qw(seabios ovmf) ],
 633         description => "Select BIOS implementation.",
 634         default => 'seabios',
 635     },
 636     vmgenid => {
 637         type => 'string',
 638         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 639         format_description => 'UUID',
 640         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 641             ." to disable explicitly.",
 642         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 643             ." value identifier to the guest OS. This allows to notify the guest operating system"
 644             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 645             ." execution or creation from a template). The guest operating system notices the"
 646             ." change, and is then able to react as appropriate by marking its copies of"
 647             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 648             ."Note that auto-creation only works when done through API/CLI create or update methods"
 649             .", but not when manually editing the config file.",
 650         default => "1 (autogenerated)",
 651         optional => 1,
 652     },
 653     hookscript => {
 654         type => 'string',
 655         format => 'pve-volume-id',
 656         optional => 1,
 657         description => "Script that will be executed during various steps in the vms lifetime.",
 658     },
 659     ivshmem => {
 660         type => 'string',
 661         format => $ivshmem_fmt,
 662         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 663             ." the host.",
 664         optional => 1,
 665     },
 666     audio0 => {
 667         type => 'string',
 668         format => $audio_fmt,
 669         description => "Configure a audio device, useful in combination with QXL/Spice.",
 670         optional => 1
 671     },
 672     spice_enhancements => {
 673         type => 'string',
 674         format => $spice_enhancements_fmt,
 675         description => "Configure additional enhancements for SPICE.",
 676         optional => 1
 677     },
 678     tags => {
 679         type => 'string', format => 'pve-tag-list',
 680         description => 'Tags of the VM. This is only meta information.',
 681         optional => 1,
 682     },
 683     rng0 => {
 684         type => 'string',
 685         format => $rng_fmt,
 686         description => "Configure a VirtIO-based Random Number Generator.",
 687         optional => 1,
 688     },
 689 };
 690
 691 my $cicustom_fmt = {
 692     meta => {
 693         type => 'string',
 694         optional => 1,
 695         description => 'Specify a custom file containing all meta data passed to the VM via"
 696             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 697         format => 'pve-volume-id',
 698         format_description => 'volume',
 699     },
 700     network => {
 701         type => 'string',
 702         optional => 1,
 703         description => 'Specify a custom file containing all network data passed to the VM via'
 704             .' cloud-init.',
 705         format => 'pve-volume-id',
 706         format_description => 'volume',
 707     },
 708     user => {
 709         type => 'string',
 710         optional => 1,
 711         description => 'Specify a custom file containing all user data passed to the VM via'
 712             .' cloud-init.',
 713         format => 'pve-volume-id',
 714         format_description => 'volume',
 715     },
 716 };
 717 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 718
 719 my $confdesc_cloudinit = {
 720     citype => {
 721         optional => 1,
 722         type => 'string',
 723         description => 'Specifies the cloud-init configuration format. The default depends on the'
 724             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 725             .' and `configdrive2` for windows.',
 726         enum => ['configdrive2', 'nocloud'],
 727     },
 728     ciuser => {
 729         optional => 1,
 730         type => 'string',
 731         description => "cloud-init: User name to change ssh keys and password for instead of the"
 732             ." image's configured default user.",
 733     },
 734     cipassword => {
 735         optional => 1,
 736         type => 'string',
 737         description => 'cloud-init: Password to assign the user. Using this is generally not'
 738             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 739             .' support hashed passwords.',
 740     },
 741     cicustom => {
 742         optional => 1,
 743         type => 'string',
 744         description => 'cloud-init: Specify custom files to replace the automatically generated'
 745             .' ones at start.',
 746         format => 'pve-qm-cicustom',
 747     },
 748     searchdomain => {
 749         optional => 1,
 750         type => 'string',
 751         description => "cloud-init: Sets DNS search domains for a container. Create will'
 752             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 753             .' are set.",
 754     },
 755     nameserver => {
 756         optional => 1,
 757         type => 'string', format => 'address-list',
 758         description => "cloud-init: Sets DNS server IP address for a container. Create will'
 759             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 760             .' are set.",
 761     },
 762     sshkeys => {
 763         optional => 1,
 764         type => 'string',
 765         format => 'urlencoded',
 766         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 767     },
 768 };
 769
 770 # what about other qemu settings ?
 771 #cpu => 'string',
 772 #machine => 'string',
 773 #fda => 'file',
 774 #fdb => 'file',
 775 #mtdblock => 'file',
 776 #sd => 'file',
 777 #pflash => 'file',
 778 #snapshot => 'bool',
 779 #bootp => 'file',
 780 ##tftp => 'dir',
 781 ##smb => 'dir',
 782 #kernel => 'file',
 783 #append => 'string',
 784 #initrd => 'file',
 785 ##soundhw => 'string',
 786
 787 while (my ($k, $v) = each %$confdesc) {
 788     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 789 }
 790
 791 my $MAX_USB_DEVICES = 5;
 792 my $MAX_NETS = 32;
 793 my $MAX_SERIAL_PORTS = 4;
 794 my $MAX_PARALLEL_PORTS = 3;
 795 my $MAX_NUMA = 8;
 796
 797 my $numa_fmt = {
 798     cpus => {
 799         type => "string",
 800         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 801         description => "CPUs accessing this NUMA node.",
 802         format_description => "id[-id];...",
 803     },
 804     memory => {
 805         type => "number",
 806         description => "Amount of memory this NUMA node provides.",
 807         optional => 1,
 808     },
 809     hostnodes => {
 810         type => "string",
 811         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 812         description => "Host NUMA nodes to use.",
 813         format_description => "id[-id];...",
 814         optional => 1,
 815     },
 816     policy => {
 817         type => 'string',
 818         enum => [qw(preferred bind interleave)],
 819         description => "NUMA allocation policy.",
 820         optional => 1,
 821     },
 822 };
 823 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 824 my $numadesc = {
 825     optional => 1,
 826     type => 'string', format => $numa_fmt,
 827     description => "NUMA topology.",
 828 };
 829 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 830
 831 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 832     $confdesc->{"numa$i"} = $numadesc;
 833 }
 834
 835 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 836                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 837                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 838 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 839
 840 my $net_fmt_bridge_descr = <<__EOD__;
 841 Bridge to attach the network device to. The Proxmox VE standard bridge
 842 is called 'vmbr0'.
 843
 844 If you do not specify a bridge, we create a kvm user (NATed) network
 845 device, which provides DHCP and DNS services. The following addresses
 846 are used:
 847
 848  10.0.2.2   Gateway
 849  10.0.2.3   DNS Server
 850  10.0.2.4   SMB Server
 851
 852 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 853 __EOD__
 854
 855 my $net_fmt = {
 856     macaddr  => get_standard_option('mac-addr', {
 857         description => "MAC address. That address must be unique withing your network. This is"
 858             ." automatically generated if not specified.",
 859     }),
 860     model => {
 861         type => 'string',
 862         description => "Network Card Model. The 'virtio' model provides the best performance with"
 863             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 864             ." best to use 'e1000'.",
 865         enum => $nic_model_list,
 866         default_key => 1,
 867     },
 868     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 869     bridge => {
 870         type => 'string',
 871         description => $net_fmt_bridge_descr,
 872         format_description => 'bridge',
 873         pattern => '[-_.\w\d]+',
 874         optional => 1,
 875     },
 876     queues => {
 877         type => 'integer',
 878         minimum => 0, maximum => 16,
 879         description => 'Number of packet queues to be used on the device.',
 880         optional => 1,
 881     },
 882     rate => {
 883         type => 'number',
 884         minimum => 0,
 885         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 886         optional => 1,
 887     },
 888     tag => {
 889         type => 'integer',
 890         minimum => 1, maximum => 4094,
 891         description => 'VLAN tag to apply to packets on this interface.',
 892         optional => 1,
 893     },
 894     trunks => {
 895         type => 'string',
 896         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 897         description => 'VLAN trunks to pass through this interface.',
 898         format_description => 'vlanid[;vlanid...]',
 899         optional => 1,
 900     },
 901     firewall => {
 902         type => 'boolean',
 903         description => 'Whether this interface should be protected by the firewall.',
 904         optional => 1,
 905     },
 906     link_down => {
 907         type => 'boolean',
 908         description => 'Whether this interface should be disconnected (like pulling the plug).',
 909         optional => 1,
 910     },
 911     mtu => {
 912         type => 'integer',
 913         minimum => 1, maximum => 65520,
 914         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 915         optional => 1,
 916     },
 917 };
 918
 919 my $netdesc = {
 920     optional => 1,
 921     type => 'string', format => $net_fmt,
 922     description => "Specify network devices.",
 923 };
 924
 925 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 926
 927 my $ipconfig_fmt = {
 928     ip => {
 929         type => 'string',
 930         format => 'pve-ipv4-config',
 931         format_description => 'IPv4Format/CIDR',
 932         description => 'IPv4 address in CIDR format.',
 933         optional => 1,
 934         default => 'dhcp',
 935     },
 936     gw => {
 937         type => 'string',
 938         format => 'ipv4',
 939         format_description => 'GatewayIPv4',
 940         description => 'Default gateway for IPv4 traffic.',
 941         optional => 1,
 942         requires => 'ip',
 943     },
 944     ip6 => {
 945         type => 'string',
 946         format => 'pve-ipv6-config',
 947         format_description => 'IPv6Format/CIDR',
 948         description => 'IPv6 address in CIDR format.',
 949         optional => 1,
 950         default => 'dhcp',
 951     },
 952     gw6 => {
 953         type => 'string',
 954         format => 'ipv6',
 955         format_description => 'GatewayIPv6',
 956         description => 'Default gateway for IPv6 traffic.',
 957         optional => 1,
 958         requires => 'ip6',
 959     },
 960 };
 961 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 962 my $ipconfigdesc = {
 963     optional => 1,
 964     type => 'string', format => 'pve-qm-ipconfig',
 965     description => <<'EODESCR',
 966 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 967
 968 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 969
 970 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
 971 gateway should be provided.
 972 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 973
 974 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
 975 dhcp on IPv4.
 976 EODESCR
 977 };
 978 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 979
 980 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 981     $confdesc->{"net$i"} = $netdesc;
 982     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 983 }
 984
 985 foreach my $key (keys %$confdesc_cloudinit) {
 986     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 987 }
 988
 989 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 990 sub verify_volume_id_or_qm_path {
 991     my ($volid, $noerr) = @_;
 992
 993     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 994         return $volid;
 995     }
 996
 997     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 998     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 999     if ($@) {
1000         return if $noerr;
1001         die $@;
1002     }
1003     return $volid;
1004 }
1005
1006 my $usb_fmt = {
1007     host => {
1008         default_key => 1,
1009         type => 'string', format => 'pve-qm-usb-device',
1010         format_description => 'HOSTUSBDEVICE|spice',
1011         description => <<EODESCR,
1012 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1013
1014  'bus-port(.port)*' (decimal numbers) or
1015  'vendor_id:product_id' (hexadeciaml numbers) or
1016  'spice'
1017
1018 You can use the 'lsusb -t' command to list existing usb devices.
1019
1020 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1021 machines - use with special care.
1022
1023 The value 'spice' can be used to add a usb redirection devices for spice.
1024 EODESCR
1025     },
1026     usb3 => {
1027         optional => 1,
1028         type => 'boolean',
1029         description => "Specifies whether if given host option is a USB3 device or port.",
1030         default => 0,
1031     },
1032 };
1033
1034 my $usbdesc = {
1035     optional => 1,
1036     type => 'string', format => $usb_fmt,
1037     description => "Configure an USB device (n is 0 to 4).",
1038 };
1039 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1040
1041 my $serialdesc = {
1042         optional => 1,
1043         type => 'string',
1044         pattern => '(/dev/.+|socket)',
1045         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1046         verbose_description =>  <<EODESCR,
1047 Create a serial device inside the VM (n is 0 to 3), and pass through a
1048 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1049 host side (use 'qm terminal' to open a terminal connection).
1050
1051 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1052 use with special care.
1053
1054 CAUTION: Experimental! User reported problems with this option.
1055 EODESCR
1056 };
1057
1058 my $paralleldesc= {
1059         optional => 1,
1060         type => 'string',
1061         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1062         description =>  "Map host parallel devices (n is 0 to 2).",
1063         verbose_description =>  <<EODESCR,
1064 Map host parallel devices (n is 0 to 2).
1065
1066 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1067 machines - use with special care.
1068
1069 CAUTION: Experimental! User reported problems with this option.
1070 EODESCR
1071 };
1072
1073 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1074     $confdesc->{"parallel$i"} = $paralleldesc;
1075 }
1076
1077 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1078     $confdesc->{"serial$i"} = $serialdesc;
1079 }
1080
1081 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1082     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1083 }
1084
1085 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1086     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1087 }
1088
1089 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1090     $confdesc->{"usb$i"} = $usbdesc;
1091 }
1092
1093 my $boot_fmt = {
1094     legacy => {
1095         optional => 1,
1096         default_key => 1,
1097         type => 'string',
1098         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1099                      . " Deprecated, use 'order=' instead.",
1100         pattern => '[acdn]{1,4}',
1101         format_description => "[acdn]{1,4}",
1102
1103         # note: this is also the fallback if boot: is not given at all
1104         default => 'cdn',
1105     },
1106     order => {
1107         optional => 1,
1108         type => 'string',
1109         format => 'pve-qm-bootdev-list',
1110         format_description => "device[;device...]",
1111         description => <<EODESC,
1112 The guest will attempt to boot from devices in the order they appear here.
1113
1114 Disks, optical drives and passed-through storage USB devices will be directly
1115 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1116 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1117
1118 Note that only devices in this list will be marked as bootable and thus loaded
1119 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1120 (e.g. software-raid), you need to specify all of them here.
1121
1122 Overrides the deprecated 'legacy=[acdn]*' value when given.
1123 EODESC
1124     },
1125 };
1126 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1127
1128 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1129 sub verify_bootdev {
1130     my ($dev, $noerr) = @_;
1131
1132     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && $dev !~ m/^efidisk/;
1133
1134     my $check = sub {
1135         my ($base) = @_;
1136         return 0 if $dev !~ m/^$base\d+$/;
1137         return 0 if !$confdesc->{$dev};
1138         return 1;
1139     };
1140
1141     return $dev if $check->("net");
1142     return $dev if $check->("usb");
1143     return $dev if $check->("hostpci");
1144
1145     return if $noerr;
1146     die "invalid boot device '$dev'\n";
1147 }
1148
1149 sub print_bootorder {
1150     my ($devs) = @_;
1151     return "" if !@$devs;
1152     my $data = { order => join(';', @$devs) };
1153     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1154 }
1155
1156 my $kvm_api_version = 0;
1157
1158 sub kvm_version {
1159     return $kvm_api_version if $kvm_api_version;
1160
1161     open my $fh, '<', '/dev/kvm' or return;
1162
1163     # 0xae00 => KVM_GET_API_VERSION
1164     $kvm_api_version = ioctl($fh, 0xae00, 0);
1165     close($fh);
1166
1167     return $kvm_api_version;
1168 }
1169
1170 my $kvm_user_version = {};
1171 my $kvm_mtime = {};
1172
1173 sub kvm_user_version {
1174     my ($binary) = @_;
1175
1176     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1177     my $st = stat($binary);
1178
1179     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1180     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1181         $cachedmtime == $st->mtime;
1182
1183     $kvm_user_version->{$binary} = 'unknown';
1184     $kvm_mtime->{$binary} = $st->mtime;
1185
1186     my $code = sub {
1187         my $line = shift;
1188         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1189             $kvm_user_version->{$binary} = $2;
1190         }
1191     };
1192
1193     eval { run_command([$binary, '--version'], outfunc => $code); };
1194     warn $@ if $@;
1195
1196     return $kvm_user_version->{$binary};
1197
1198 }
1199 my sub extract_version {
1200     my ($machine_type, $version) = @_;
1201     $version = kvm_user_version() if !defined($version);
1202     PVE::QemuServer::Machine::extract_version($machine_type, $version)
1203 }
1204
1205 sub kernel_has_vhost_net {
1206     return -c '/dev/vhost-net';
1207 }
1208
1209 sub option_exists {
1210     my $key = shift;
1211     return defined($confdesc->{$key});
1212 }
1213
1214 my $cdrom_path;
1215 sub get_cdrom_path {
1216
1217     return  $cdrom_path if $cdrom_path;
1218
1219     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1220     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1221     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1222 }
1223
1224 sub get_iso_path {
1225     my ($storecfg, $vmid, $cdrom) = @_;
1226
1227     if ($cdrom eq 'cdrom') {
1228         return get_cdrom_path();
1229     } elsif ($cdrom eq 'none') {
1230         return '';
1231     } elsif ($cdrom =~ m|^/|) {
1232         return $cdrom;
1233     } else {
1234         return PVE::Storage::path($storecfg, $cdrom);
1235     }
1236 }
1237
1238 # try to convert old style file names to volume IDs
1239 sub filename_to_volume_id {
1240     my ($vmid, $file, $media) = @_;
1241
1242      if (!($file eq 'none' || $file eq 'cdrom' ||
1243           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1244
1245         return if $file =~ m|/|;
1246
1247         if ($media && $media eq 'cdrom') {
1248             $file = "local:iso/$file";
1249         } else {
1250             $file = "local:$vmid/$file";
1251         }
1252     }
1253
1254     return $file;
1255 }
1256
1257 sub verify_media_type {
1258     my ($opt, $vtype, $media) = @_;
1259
1260     return if !$media;
1261
1262     my $etype;
1263     if ($media eq 'disk') {
1264         $etype = 'images';
1265     } elsif ($media eq 'cdrom') {
1266         $etype = 'iso';
1267     } else {
1268         die "internal error";
1269     }
1270
1271     return if ($vtype eq $etype);
1272
1273     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1274 }
1275
1276 sub cleanup_drive_path {
1277     my ($opt, $storecfg, $drive) = @_;
1278
1279     # try to convert filesystem paths to volume IDs
1280
1281     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1282         ($drive->{file} !~ m|^/dev/.+|) &&
1283         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1284         ($drive->{file} !~ m/^\d+$/)) {
1285         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1286         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1287             if !$vtype;
1288         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1289         verify_media_type($opt, $vtype, $drive->{media});
1290         $drive->{file} = $volid;
1291     }
1292
1293     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1294 }
1295
1296 sub parse_hotplug_features {
1297     my ($data) = @_;
1298
1299     my $res = {};
1300
1301     return $res if $data eq '0';
1302
1303     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1304
1305     foreach my $feature (PVE::Tools::split_list($data)) {
1306         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1307             $res->{$1} = 1;
1308         } else {
1309             die "invalid hotplug feature '$feature'\n";
1310         }
1311     }
1312     return $res;
1313 }
1314
1315 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1316 sub pve_verify_hotplug_features {
1317     my ($value, $noerr) = @_;
1318
1319     return $value if parse_hotplug_features($value);
1320
1321     return if $noerr;
1322
1323     die "unable to parse hotplug option\n";
1324 }
1325
1326 sub scsi_inquiry {
1327     my($fh, $noerr) = @_;
1328
1329     my $SG_IO = 0x2285;
1330     my $SG_GET_VERSION_NUM = 0x2282;
1331
1332     my $versionbuf = "\x00" x 8;
1333     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1334     if (!$ret) {
1335         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1336         return;
1337     }
1338     my $version = unpack("I", $versionbuf);
1339     if ($version < 30000) {
1340         die "scsi generic interface too old\n"  if !$noerr;
1341         return;
1342     }
1343
1344     my $buf = "\x00" x 36;
1345     my $sensebuf = "\x00" x 8;
1346     my $cmd = pack("C x3 C x1", 0x12, 36);
1347
1348     # see /usr/include/scsi/sg.h
1349     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1350
1351     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1352                       length($sensebuf), 0, length($buf), $buf,
1353                       $cmd, $sensebuf, 6000);
1354
1355     $ret = ioctl($fh, $SG_IO, $packet);
1356     if (!$ret) {
1357         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1358         return;
1359     }
1360
1361     my @res = unpack($sg_io_hdr_t, $packet);
1362     if ($res[17] || $res[18]) {
1363         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1364         return;
1365     }
1366
1367     my $res = {};
1368     (my $byte0, my $byte1, $res->{vendor},
1369      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1370
1371     $res->{removable} = $byte1 & 128 ? 1 : 0;
1372     $res->{type} = $byte0 & 31;
1373
1374     return $res;
1375 }
1376
1377 sub path_is_scsi {
1378     my ($path) = @_;
1379
1380     my $fh = IO::File->new("+<$path") || return;
1381     my $res = scsi_inquiry($fh, 1);
1382     close($fh);
1383
1384     return $res;
1385 }
1386
1387 sub print_tabletdevice_full {
1388     my ($conf, $arch) = @_;
1389
1390     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1391
1392     # we use uhci for old VMs because tablet driver was buggy in older qemu
1393     my $usbbus;
1394     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1395         $usbbus = 'ehci';
1396     } else {
1397         $usbbus = 'uhci';
1398     }
1399
1400     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1401 }
1402
1403 sub print_keyboarddevice_full {
1404     my ($conf, $arch, $machine) = @_;
1405
1406     return if $arch ne 'aarch64';
1407
1408     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1409 }
1410
1411 sub print_drivedevice_full {
1412     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1413
1414     my $device = '';
1415     my $maxdev = 0;
1416
1417     my $drive_id = "$drive->{interface}$drive->{index}";
1418     if ($drive->{interface} eq 'virtio') {
1419         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1420         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1421         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1422     } elsif ($drive->{interface} eq 'scsi') {
1423
1424         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1425         my $unit = $drive->{index} % $maxdev;
1426         my $devicetype = 'hd';
1427         my $path = '';
1428         if (drive_is_cdrom($drive)) {
1429             $devicetype = 'cd';
1430         } else {
1431             if ($drive->{file} =~ m|^/|) {
1432                 $path = $drive->{file};
1433                 if (my $info = path_is_scsi($path)) {
1434                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1435                         $devicetype = 'block';
1436                     } elsif ($info->{type} == 1) { # tape
1437                         $devicetype = 'generic';
1438                     }
1439                 }
1440             } else {
1441                  $path = PVE::Storage::path($storecfg, $drive->{file});
1442             }
1443
1444             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1445             my $version = extract_version($machine_type, kvm_user_version());
1446             if ($path =~ m/^iscsi\:\/\// &&
1447                !min_version($version, 4, 1)) {
1448                 $devicetype = 'generic';
1449             }
1450         }
1451
1452         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1453             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1454         } else {
1455             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1456                 .",lun=$drive->{index}";
1457         }
1458         $device .= ",drive=drive-$drive_id,id=$drive_id";
1459
1460         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1461             $device .= ",rotation_rate=1";
1462         }
1463         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1464
1465     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1466         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1467         my $controller = int($drive->{index} / $maxdev);
1468         my $unit = $drive->{index} % $maxdev;
1469         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1470
1471         $device = "ide-$devicetype";
1472         if ($drive->{interface} eq 'ide') {
1473             $device .= ",bus=ide.$controller,unit=$unit";
1474         } else {
1475             $device .= ",bus=ahci$controller.$unit";
1476         }
1477         $device .= ",drive=drive-$drive_id,id=$drive_id";
1478
1479         if ($devicetype eq 'hd') {
1480             if (my $model = $drive->{model}) {
1481                 $model = URI::Escape::uri_unescape($model);
1482                 $device .= ",model=$model";
1483             }
1484             if ($drive->{ssd}) {
1485                 $device .= ",rotation_rate=1";
1486             }
1487         }
1488         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1489     } elsif ($drive->{interface} eq 'usb') {
1490         die "implement me";
1491         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1492     } else {
1493         die "unsupported interface type";
1494     }
1495
1496     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1497
1498     if (my $serial = $drive->{serial}) {
1499         $serial = URI::Escape::uri_unescape($serial);
1500         $device .= ",serial=$serial";
1501     }
1502
1503
1504     return $device;
1505 }
1506
1507 sub get_initiator_name {
1508     my $initiator;
1509
1510     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1511     while (defined(my $line = <$fh>)) {
1512         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1513         $initiator = $1;
1514         last;
1515     }
1516     $fh->close();
1517
1518     return $initiator;
1519 }
1520
1521 sub print_drive_commandline_full {
1522     my ($storecfg, $vmid, $drive) = @_;
1523
1524     my $path;
1525     my $volid = $drive->{file};
1526     my $format;
1527
1528     if (drive_is_cdrom($drive)) {
1529         $path = get_iso_path($storecfg, $vmid, $volid);
1530     } else {
1531         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1532         if ($storeid) {
1533             $path = PVE::Storage::path($storecfg, $volid);
1534             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1535             $format = qemu_img_format($scfg, $volname);
1536         } else {
1537             $path = $volid;
1538             $format = "raw";
1539         }
1540    }
1541
1542     my $opts = '';
1543     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1544     foreach my $o (@qemu_drive_options) {
1545         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1546     }
1547
1548     # snapshot only accepts on|off
1549     if (defined($drive->{snapshot})) {
1550         my $v = $drive->{snapshot} ? 'on' : 'off';
1551         $opts .= ",snapshot=$v";
1552     }
1553
1554     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1555         my ($dir, $qmpname) = @$type;
1556         if (my $v = $drive->{"mbps$dir"}) {
1557             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1558         }
1559         if (my $v = $drive->{"mbps${dir}_max"}) {
1560             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1561         }
1562         if (my $v = $drive->{"bps${dir}_max_length"}) {
1563             $opts .= ",throttling.bps$qmpname-max-length=$v";
1564         }
1565         if (my $v = $drive->{"iops${dir}"}) {
1566             $opts .= ",throttling.iops$qmpname=$v";
1567         }
1568         if (my $v = $drive->{"iops${dir}_max"}) {
1569             $opts .= ",throttling.iops$qmpname-max=$v";
1570         }
1571         if (my $v = $drive->{"iops${dir}_max_length"}) {
1572             $opts .= ",throttling.iops$qmpname-max-length=$v";
1573         }
1574     }
1575
1576     $opts .= ",format=$format" if $format && !$drive->{format};
1577
1578     my $cache_direct = 0;
1579
1580     if (my $cache = $drive->{cache}) {
1581         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1582     } elsif (!drive_is_cdrom($drive)) {
1583         $opts .= ",cache=none";
1584         $cache_direct = 1;
1585     }
1586
1587     # aio native works only with O_DIRECT
1588     if (!$drive->{aio}) {
1589         if($cache_direct) {
1590             $opts .= ",aio=native";
1591         } else {
1592             $opts .= ",aio=threads";
1593         }
1594     }
1595
1596     if (!drive_is_cdrom($drive)) {
1597         my $detectzeroes;
1598         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1599             $detectzeroes = 'off';
1600         } elsif ($drive->{discard}) {
1601             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1602         } else {
1603             # This used to be our default with discard not being specified:
1604             $detectzeroes = 'on';
1605         }
1606         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1607     }
1608
1609     my $pathinfo = $path ? "file=$path," : '';
1610
1611     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1612 }
1613
1614 sub print_netdevice_full {
1615     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1616
1617     my $device = $net->{model};
1618     if ($net->{model} eq 'virtio') {
1619          $device = 'virtio-net-pci';
1620      };
1621
1622     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1623     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1624     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1625         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1626         # and out of each queue plus one config interrupt and control vector queue
1627         my $vectors = $net->{queues} * 2 + 2;
1628         $tmpstr .= ",vectors=$vectors,mq=on";
1629     }
1630     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1631
1632     if (my $mtu = $net->{mtu}) {
1633         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1634             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1635             if ($mtu == 1) {
1636                  $mtu = $bridge_mtu;
1637             } elsif ($mtu < 576) {
1638                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1639             } elsif ($mtu > $bridge_mtu) {
1640                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1641             }
1642             $tmpstr .= ",host_mtu=$mtu";
1643         } else {
1644             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1645         }
1646     }
1647
1648     if ($use_old_bios_files) {
1649         my $romfile;
1650         if ($device eq 'virtio-net-pci') {
1651             $romfile = 'pxe-virtio.rom';
1652         } elsif ($device eq 'e1000') {
1653             $romfile = 'pxe-e1000.rom';
1654         } elsif ($device eq 'ne2k') {
1655             $romfile = 'pxe-ne2k_pci.rom';
1656         } elsif ($device eq 'pcnet') {
1657             $romfile = 'pxe-pcnet.rom';
1658         } elsif ($device eq 'rtl8139') {
1659             $romfile = 'pxe-rtl8139.rom';
1660         }
1661         $tmpstr .= ",romfile=$romfile" if $romfile;
1662     }
1663
1664     return $tmpstr;
1665 }
1666
1667 sub print_netdev_full {
1668     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1669
1670     my $i = '';
1671     if ($netid =~ m/^net(\d+)$/) {
1672         $i = int($1);
1673     }
1674
1675     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1676
1677     my $ifname = "tap${vmid}i$i";
1678
1679     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1680     die "interface name '$ifname' is too long (max 15 character)\n"
1681         if length($ifname) >= 16;
1682
1683     my $vhostparam = '';
1684     if (is_native($arch)) {
1685         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1686     }
1687
1688     my $vmname = $conf->{name} || "vm$vmid";
1689
1690     my $netdev = "";
1691     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1692
1693     if ($net->{bridge}) {
1694         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1695             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1696     } else {
1697         $netdev = "type=user,id=$netid,hostname=$vmname";
1698     }
1699
1700     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1701
1702     return $netdev;
1703 }
1704
1705 my $vga_map = {
1706     'cirrus' => 'cirrus-vga',
1707     'std' => 'VGA',
1708     'vmware' => 'vmware-svga',
1709     'virtio' => 'virtio-vga',
1710 };
1711
1712 sub print_vga_device {
1713     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1714
1715     my $type = $vga_map->{$vga->{type}};
1716     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1717         $type = 'virtio-gpu';
1718     }
1719     my $vgamem_mb = $vga->{memory};
1720
1721     my $max_outputs = '';
1722     if ($qxlnum) {
1723         $type = $id ? 'qxl' : 'qxl-vga';
1724
1725         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1726             # set max outputs so linux can have up to 4 qxl displays with one device
1727             if (min_version($machine_version, 4, 1)) {
1728                 $max_outputs = ",max_outputs=4";
1729             }
1730         }
1731     }
1732
1733     die "no devicetype for $vga->{type}\n" if !$type;
1734
1735     my $memory = "";
1736     if ($vgamem_mb) {
1737         if ($vga->{type} eq 'virtio') {
1738             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1739             $memory = ",max_hostmem=$bytes";
1740         } elsif ($qxlnum) {
1741             # from https://www.spice-space.org/multiple-monitors.html
1742             $memory = ",vgamem_mb=$vga->{memory}";
1743             my $ram = $vgamem_mb * 4;
1744             my $vram = $vgamem_mb * 2;
1745             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1746         } else {
1747             $memory = ",vgamem_mb=$vga->{memory}";
1748         }
1749     } elsif ($qxlnum && $id) {
1750         $memory = ",ram_size=67108864,vram_size=33554432";
1751     }
1752
1753     my $edidoff = "";
1754     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1755         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1756     }
1757
1758     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1759     my $vgaid = "vga" . ($id // '');
1760     my $pciaddr;
1761
1762     if ($q35 && $vgaid eq 'vga') {
1763         # the first display uses pcie.0 bus on q35 machines
1764         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1765     } else {
1766         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1767     }
1768
1769     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1770 }
1771
1772 sub parse_number_sets {
1773     my ($set) = @_;
1774     my $res = [];
1775     foreach my $part (split(/;/, $set)) {
1776         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1777             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1778             push @$res, [ $1, $2 ];
1779         } else {
1780             die "invalid range: $part\n";
1781         }
1782     }
1783     return $res;
1784 }
1785
1786 sub parse_numa {
1787     my ($data) = @_;
1788
1789     my $res = parse_property_string($numa_fmt, $data);
1790     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1791     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1792     return $res;
1793 }
1794
1795 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1796 sub parse_net {
1797     my ($data) = @_;
1798
1799     my $res = eval { parse_property_string($net_fmt, $data) };
1800     if ($@) {
1801         warn $@;
1802         return;
1803     }
1804     if (!defined($res->{macaddr})) {
1805         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1806         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1807     }
1808     return $res;
1809 }
1810
1811 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1812 sub parse_ipconfig {
1813     my ($data) = @_;
1814
1815     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1816     if ($@) {
1817         warn $@;
1818         return;
1819     }
1820
1821     if ($res->{gw} && !$res->{ip}) {
1822         warn 'gateway specified without specifying an IP address';
1823         return;
1824     }
1825     if ($res->{gw6} && !$res->{ip6}) {
1826         warn 'IPv6 gateway specified without specifying an IPv6 address';
1827         return;
1828     }
1829     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1830         warn 'gateway specified together with DHCP';
1831         return;
1832     }
1833     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1834         # gw6 + auto/dhcp
1835         warn "IPv6 gateway specified together with $res->{ip6} address";
1836         return;
1837     }
1838
1839     if (!$res->{ip} && !$res->{ip6}) {
1840         return { ip => 'dhcp', ip6 => 'dhcp' };
1841     }
1842
1843     return $res;
1844 }
1845
1846 sub print_net {
1847     my $net = shift;
1848
1849     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1850 }
1851
1852 sub add_random_macs {
1853     my ($settings) = @_;
1854
1855     foreach my $opt (keys %$settings) {
1856         next if $opt !~ m/^net(\d+)$/;
1857         my $net = parse_net($settings->{$opt});
1858         next if !$net;
1859         $settings->{$opt} = print_net($net);
1860     }
1861 }
1862
1863 sub vm_is_volid_owner {
1864     my ($storecfg, $vmid, $volid) = @_;
1865
1866     if ($volid !~  m|^/|) {
1867         my ($path, $owner);
1868         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1869         if ($owner && ($owner == $vmid)) {
1870             return 1;
1871         }
1872     }
1873
1874     return;
1875 }
1876
1877 sub vmconfig_register_unused_drive {
1878     my ($storecfg, $vmid, $conf, $drive) = @_;
1879
1880     if (drive_is_cloudinit($drive)) {
1881         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1882         warn $@ if $@;
1883     } elsif (!drive_is_cdrom($drive)) {
1884         my $volid = $drive->{file};
1885         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1886             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1887         }
1888     }
1889 }
1890
1891 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1892 my $smbios1_fmt = {
1893     uuid => {
1894         type => 'string',
1895         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1896         format_description => 'UUID',
1897         description => "Set SMBIOS1 UUID.",
1898         optional => 1,
1899     },
1900     version => {
1901         type => 'string',
1902         pattern => '[A-Za-z0-9+\/]+={0,2}',
1903         format_description => 'Base64 encoded string',
1904         description => "Set SMBIOS1 version.",
1905         optional => 1,
1906     },
1907     serial => {
1908         type => 'string',
1909         pattern => '[A-Za-z0-9+\/]+={0,2}',
1910         format_description => 'Base64 encoded string',
1911         description => "Set SMBIOS1 serial number.",
1912         optional => 1,
1913     },
1914     manufacturer => {
1915         type => 'string',
1916         pattern => '[A-Za-z0-9+\/]+={0,2}',
1917         format_description => 'Base64 encoded string',
1918         description => "Set SMBIOS1 manufacturer.",
1919         optional => 1,
1920     },
1921     product => {
1922         type => 'string',
1923         pattern => '[A-Za-z0-9+\/]+={0,2}',
1924         format_description => 'Base64 encoded string',
1925         description => "Set SMBIOS1 product ID.",
1926         optional => 1,
1927     },
1928     sku => {
1929         type => 'string',
1930         pattern => '[A-Za-z0-9+\/]+={0,2}',
1931         format_description => 'Base64 encoded string',
1932         description => "Set SMBIOS1 SKU string.",
1933         optional => 1,
1934     },
1935     family => {
1936         type => 'string',
1937         pattern => '[A-Za-z0-9+\/]+={0,2}',
1938         format_description => 'Base64 encoded string',
1939         description => "Set SMBIOS1 family string.",
1940         optional => 1,
1941     },
1942     base64 => {
1943         type => 'boolean',
1944         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
1945         optional => 1,
1946     },
1947 };
1948
1949 sub parse_smbios1 {
1950     my ($data) = @_;
1951
1952     my $res = eval { parse_property_string($smbios1_fmt, $data) };
1953     warn $@ if $@;
1954     return $res;
1955 }
1956
1957 sub print_smbios1 {
1958     my ($smbios1) = @_;
1959     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1960 }
1961
1962 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1963
1964 sub parse_watchdog {
1965     my ($value) = @_;
1966
1967     return if !$value;
1968
1969     my $res = eval { parse_property_string($watchdog_fmt, $value) };
1970     warn $@ if $@;
1971     return $res;
1972 }
1973
1974 sub parse_guest_agent {
1975     my ($value) = @_;
1976
1977     return {} if !defined($value->{agent});
1978
1979     my $res = eval { parse_property_string($agent_fmt, $value->{agent}) };
1980     warn $@ if $@;
1981
1982     # if the agent is disabled ignore the other potentially set properties
1983     return {} if !$res->{enabled};
1984     return $res;
1985 }
1986
1987 sub parse_vga {
1988     my ($value) = @_;
1989
1990     return {} if !$value;
1991     my $res = eval { parse_property_string($vga_fmt, $value) };
1992     warn $@ if $@;
1993     return $res;
1994 }
1995
1996 sub parse_rng {
1997     my ($value) = @_;
1998
1999     return if !$value;
2000
2001     my $res = eval { parse_property_string($rng_fmt, $value) };
2002     warn $@ if $@;
2003     return $res;
2004 }
2005
2006 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2007 sub verify_usb_device {
2008     my ($value, $noerr) = @_;
2009
2010     return $value if parse_usb_device($value);
2011
2012     return if $noerr;
2013
2014     die "unable to parse usb device\n";
2015 }
2016
2017 # add JSON properties for create and set function
2018 sub json_config_properties {
2019     my $prop = shift;
2020
2021     foreach my $opt (keys %$confdesc) {
2022         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' ||
2023             $opt eq 'runningmachine' || $opt eq 'runningcpu';
2024         $prop->{$opt} = $confdesc->{$opt};
2025     }
2026
2027     return $prop;
2028 }
2029
2030 # return copy of $confdesc_cloudinit to generate documentation
2031 sub cloudinit_config_properties {
2032
2033     return dclone($confdesc_cloudinit);
2034 }
2035
2036 sub check_type {
2037     my ($key, $value) = @_;
2038
2039     die "unknown setting '$key'\n" if !$confdesc->{$key};
2040
2041     my $type = $confdesc->{$key}->{type};
2042
2043     if (!defined($value)) {
2044         die "got undefined value\n";
2045     }
2046
2047     if ($value =~ m/[\n\r]/) {
2048         die "property contains a line feed\n";
2049     }
2050
2051     if ($type eq 'boolean') {
2052         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2053         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2054         die "type check ('boolean') failed - got '$value'\n";
2055     } elsif ($type eq 'integer') {
2056         return int($1) if $value =~ m/^(\d+)$/;
2057         die "type check ('integer') failed - got '$value'\n";
2058     } elsif ($type eq 'number') {
2059         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2060         die "type check ('number') failed - got '$value'\n";
2061     } elsif ($type eq 'string') {
2062         if (my $fmt = $confdesc->{$key}->{format}) {
2063             PVE::JSONSchema::check_format($fmt, $value);
2064             return $value;
2065         }
2066         $value =~ s/^\"(.*)\"$/$1/;
2067         return $value;
2068     } else {
2069         die "internal error"
2070     }
2071 }
2072
2073 sub destroy_vm {
2074     my ($storecfg, $vmid, $skiplock, $replacement_conf) = @_;
2075
2076     my $conf = PVE::QemuConfig->load_config($vmid);
2077
2078     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2079
2080     if ($conf->{template}) {
2081         # check if any base image is still used by a linked clone
2082         PVE::QemuConfig->foreach_volume($conf, sub {
2083                 my ($ds, $drive) = @_;
2084                 return if drive_is_cdrom($drive);
2085
2086                 my $volid = $drive->{file};
2087                 return if !$volid || $volid =~ m|^/|;
2088
2089                 die "base volume '$volid' is still in use by linked cloned\n"
2090                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2091
2092         });
2093     }
2094
2095     # only remove disks owned by this VM
2096     PVE::QemuConfig->foreach_volume($conf, sub {
2097         my ($ds, $drive) = @_;
2098         return if drive_is_cdrom($drive, 1);
2099
2100         my $volid = $drive->{file};
2101         return if !$volid || $volid =~ m|^/|;
2102
2103         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2104         return if !$path || !$owner || ($owner != $vmid);
2105
2106         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2107         warn "Could not remove disk '$volid', check manually: $@" if $@;
2108     });
2109
2110     # also remove unused disk
2111     my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2112     PVE::Storage::foreach_volid($vmdisks, sub {
2113         my ($volid, $sid, $volname, $d) = @_;
2114         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2115         warn $@ if $@;
2116     });
2117
2118     if (defined $replacement_conf) {
2119         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2120     } else {
2121         PVE::QemuConfig->destroy_config($vmid);
2122     }
2123 }
2124
2125 sub parse_vm_config {
2126     my ($filename, $raw) = @_;
2127
2128     return if !defined($raw);
2129
2130     my $res = {
2131         digest => Digest::SHA::sha1_hex($raw),
2132         snapshots => {},
2133         pending => {},
2134     };
2135
2136     $filename =~ m|/qemu-server/(\d+)\.conf$|
2137         || die "got strange filename '$filename'";
2138
2139     my $vmid = $1;
2140
2141     my $conf = $res;
2142     my $descr;
2143     my $section = '';
2144
2145     my @lines = split(/\n/, $raw);
2146     foreach my $line (@lines) {
2147         next if $line =~ m/^\s*$/;
2148
2149         if ($line =~ m/^\[PENDING\]\s*$/i) {
2150             $section = 'pending';
2151             if (defined($descr)) {
2152                 $descr =~ s/\s+$//;
2153                 $conf->{description} = $descr;
2154             }
2155             $descr = undef;
2156             $conf = $res->{$section} = {};
2157             next;
2158
2159         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2160             $section = $1;
2161             if (defined($descr)) {
2162                 $descr =~ s/\s+$//;
2163                 $conf->{description} = $descr;
2164             }
2165             $descr = undef;
2166             $conf = $res->{snapshots}->{$section} = {};
2167             next;
2168         }
2169
2170         if ($line =~ m/^\#(.*)\s*$/) {
2171             $descr = '' if !defined($descr);
2172             $descr .= PVE::Tools::decode_text($1) . "\n";
2173             next;
2174         }
2175
2176         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2177             $descr = '' if !defined($descr);
2178             $descr .= PVE::Tools::decode_text($2);
2179         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2180             $conf->{snapstate} = $1;
2181         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2182             my $key = $1;
2183             my $value = $2;
2184             $conf->{$key} = $value;
2185         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2186             my $value = $1;
2187             if ($section eq 'pending') {
2188                 $conf->{delete} = $value; # we parse this later
2189             } else {
2190                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2191             }
2192         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2193             my $key = $1;
2194             my $value = $2;
2195             eval { $value = check_type($key, $value); };
2196             if ($@) {
2197                 warn "vm $vmid - unable to parse value of '$key' - $@";
2198             } else {
2199                 $key = 'ide2' if $key eq 'cdrom';
2200                 my $fmt = $confdesc->{$key}->{format};
2201                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2202                     my $v = parse_drive($key, $value);
2203                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2204                         $v->{file} = $volid;
2205                         $value = print_drive($v);
2206                     } else {
2207                         warn "vm $vmid - unable to parse value of '$key'\n";
2208                         next;
2209                     }
2210                 }
2211
2212                 $conf->{$key} = $value;
2213             }
2214         }
2215     }
2216
2217     if (defined($descr)) {
2218         $descr =~ s/\s+$//;
2219         $conf->{description} = $descr;
2220     }
2221     delete $res->{snapstate}; # just to be sure
2222
2223     return $res;
2224 }
2225
2226 sub write_vm_config {
2227     my ($filename, $conf) = @_;
2228
2229     delete $conf->{snapstate}; # just to be sure
2230
2231     if ($conf->{cdrom}) {
2232         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2233         $conf->{ide2} = $conf->{cdrom};
2234         delete $conf->{cdrom};
2235     }
2236
2237     # we do not use 'smp' any longer
2238     if ($conf->{sockets}) {
2239         delete $conf->{smp};
2240     } elsif ($conf->{smp}) {
2241         $conf->{sockets} = $conf->{smp};
2242         delete $conf->{cores};
2243         delete $conf->{smp};
2244     }
2245
2246     my $used_volids = {};
2247
2248     my $cleanup_config = sub {
2249         my ($cref, $pending, $snapname) = @_;
2250
2251         foreach my $key (keys %$cref) {
2252             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2253                 $key eq 'snapstate' || $key eq 'pending';
2254             my $value = $cref->{$key};
2255             if ($key eq 'delete') {
2256                 die "propertry 'delete' is only allowed in [PENDING]\n"
2257                     if !$pending;
2258                 # fixme: check syntax?
2259                 next;
2260             }
2261             eval { $value = check_type($key, $value); };
2262             die "unable to parse value of '$key' - $@" if $@;
2263
2264             $cref->{$key} = $value;
2265
2266             if (!$snapname && is_valid_drivename($key)) {
2267                 my $drive = parse_drive($key, $value);
2268                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2269             }
2270         }
2271     };
2272
2273     &$cleanup_config($conf);
2274
2275     &$cleanup_config($conf->{pending}, 1);
2276
2277     foreach my $snapname (keys %{$conf->{snapshots}}) {
2278         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2279         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2280     }
2281
2282     # remove 'unusedX' settings if we re-add a volume
2283     foreach my $key (keys %$conf) {
2284         my $value = $conf->{$key};
2285         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2286             delete $conf->{$key};
2287         }
2288     }
2289
2290     my $generate_raw_config = sub {
2291         my ($conf, $pending) = @_;
2292
2293         my $raw = '';
2294
2295         # add description as comment to top of file
2296         if (defined(my $descr = $conf->{description})) {
2297             if ($descr) {
2298                 foreach my $cl (split(/\n/, $descr)) {
2299                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2300                 }
2301             } else {
2302                 $raw .= "#\n" if $pending;
2303             }
2304         }
2305
2306         foreach my $key (sort keys %$conf) {
2307             next if $key =~ /^(digest|description|pending|snapshots)$/;
2308             $raw .= "$key: $conf->{$key}\n";
2309         }
2310         return $raw;
2311     };
2312
2313     my $raw = &$generate_raw_config($conf);
2314
2315     if (scalar(keys %{$conf->{pending}})){
2316         $raw .= "\n[PENDING]\n";
2317         $raw .= &$generate_raw_config($conf->{pending}, 1);
2318     }
2319
2320     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2321         $raw .= "\n[$snapname]\n";
2322         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2323     }
2324
2325     return $raw;
2326 }
2327
2328 sub load_defaults {
2329
2330     my $res = {};
2331
2332     # we use static defaults from our JSON schema configuration
2333     foreach my $key (keys %$confdesc) {
2334         if (defined(my $default = $confdesc->{$key}->{default})) {
2335             $res->{$key} = $default;
2336         }
2337     }
2338
2339     return $res;
2340 }
2341
2342 sub config_list {
2343     my $vmlist = PVE::Cluster::get_vmlist();
2344     my $res = {};
2345     return $res if !$vmlist || !$vmlist->{ids};
2346     my $ids = $vmlist->{ids};
2347     my $nodename = nodename();
2348
2349     foreach my $vmid (keys %$ids) {
2350         my $d = $ids->{$vmid};
2351         next if !$d->{node} || $d->{node} ne $nodename;
2352         next if !$d->{type} || $d->{type} ne 'qemu';
2353         $res->{$vmid}->{exists} = 1;
2354     }
2355     return $res;
2356 }
2357
2358 # test if VM uses local resources (to prevent migration)
2359 sub check_local_resources {
2360     my ($conf, $noerr) = @_;
2361
2362     my @loc_res = ();
2363
2364     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2365     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2366
2367     push @loc_res, "ivshmem" if $conf->{ivshmem};
2368
2369     foreach my $k (keys %$conf) {
2370         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2371         # sockets are safe: they will recreated be on the target side post-migrate
2372         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2373         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2374     }
2375
2376     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2377
2378     return \@loc_res;
2379 }
2380
2381 # check if used storages are available on all nodes (use by migrate)
2382 sub check_storage_availability {
2383     my ($storecfg, $conf, $node) = @_;
2384
2385     PVE::QemuConfig->foreach_volume($conf, sub {
2386         my ($ds, $drive) = @_;
2387
2388         my $volid = $drive->{file};
2389         return if !$volid;
2390
2391         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2392         return if !$sid;
2393
2394         # check if storage is available on both nodes
2395         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2396         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2397    });
2398 }
2399
2400 # list nodes where all VM images are available (used by has_feature API)
2401 sub shared_nodes {
2402     my ($conf, $storecfg) = @_;
2403
2404     my $nodelist = PVE::Cluster::get_nodelist();
2405     my $nodehash = { map { $_ => 1 } @$nodelist };
2406     my $nodename = nodename();
2407
2408     PVE::QemuConfig->foreach_volume($conf, sub {
2409         my ($ds, $drive) = @_;
2410
2411         my $volid = $drive->{file};
2412         return if !$volid;
2413
2414         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2415         if ($storeid) {
2416             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2417             if ($scfg->{disable}) {
2418                 $nodehash = {};
2419             } elsif (my $avail = $scfg->{nodes}) {
2420                 foreach my $node (keys %$nodehash) {
2421                     delete $nodehash->{$node} if !$avail->{$node};
2422                 }
2423             } elsif (!$scfg->{shared}) {
2424                 foreach my $node (keys %$nodehash) {
2425                     delete $nodehash->{$node} if $node ne $nodename
2426                 }
2427             }
2428         }
2429     });
2430
2431     return $nodehash
2432 }
2433
2434 sub check_local_storage_availability {
2435     my ($conf, $storecfg) = @_;
2436
2437     my $nodelist = PVE::Cluster::get_nodelist();
2438     my $nodehash = { map { $_ => {} } @$nodelist };
2439
2440     PVE::QemuConfig->foreach_volume($conf, sub {
2441         my ($ds, $drive) = @_;
2442
2443         my $volid = $drive->{file};
2444         return if !$volid;
2445
2446         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2447         if ($storeid) {
2448             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2449
2450             if ($scfg->{disable}) {
2451                 foreach my $node (keys %$nodehash) {
2452                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2453                 }
2454             } elsif (my $avail = $scfg->{nodes}) {
2455                 foreach my $node (keys %$nodehash) {
2456                     if (!$avail->{$node}) {
2457                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2458                     }
2459                 }
2460             }
2461         }
2462     });
2463
2464     foreach my $node (values %$nodehash) {
2465         if (my $unavail = $node->{unavailable_storages}) {
2466             $node->{unavailable_storages} = [ sort keys %$unavail ];
2467         }
2468     }
2469
2470     return $nodehash
2471 }
2472
2473 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2474 sub check_running {
2475     my ($vmid, $nocheck, $node) = @_;
2476
2477     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2478     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2479 }
2480
2481 sub vzlist {
2482
2483     my $vzlist = config_list();
2484
2485     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2486
2487     while (defined(my $de = $fd->read)) {
2488         next if $de !~ m/^(\d+)\.pid$/;
2489         my $vmid = $1;
2490         next if !defined($vzlist->{$vmid});
2491         if (my $pid = check_running($vmid)) {
2492             $vzlist->{$vmid}->{pid} = $pid;
2493         }
2494     }
2495
2496     return $vzlist;
2497 }
2498
2499 our $vmstatus_return_properties = {
2500     vmid => get_standard_option('pve-vmid'),
2501     status => {
2502         description => "Qemu process status.",
2503         type => 'string',
2504         enum => ['stopped', 'running'],
2505     },
2506     maxmem => {
2507         description => "Maximum memory in bytes.",
2508         type => 'integer',
2509         optional => 1,
2510         renderer => 'bytes',
2511     },
2512     maxdisk => {
2513         description => "Root disk size in bytes.",
2514         type => 'integer',
2515         optional => 1,
2516         renderer => 'bytes',
2517     },
2518     name => {
2519         description => "VM name.",
2520         type => 'string',
2521         optional => 1,
2522     },
2523     qmpstatus => {
2524         description => "Qemu QMP agent status.",
2525         type => 'string',
2526         optional => 1,
2527     },
2528     pid => {
2529         description => "PID of running qemu process.",
2530         type => 'integer',
2531         optional => 1,
2532     },
2533     uptime => {
2534         description => "Uptime.",
2535         type => 'integer',
2536         optional => 1,
2537         renderer => 'duration',
2538     },
2539     cpus => {
2540         description => "Maximum usable CPUs.",
2541         type => 'number',
2542         optional => 1,
2543     },
2544     lock => {
2545         description => "The current config lock, if any.",
2546         type => 'string',
2547         optional => 1,
2548     },
2549     tags => {
2550         description  => "The current configured tags, if any",
2551         type => 'string',
2552         optional => 1,
2553     },
2554 };
2555
2556 my $last_proc_pid_stat;
2557
2558 # get VM status information
2559 # This must be fast and should not block ($full == false)
2560 # We only query KVM using QMP if $full == true (this can be slow)
2561 sub vmstatus {
2562     my ($opt_vmid, $full) = @_;
2563
2564     my $res = {};
2565
2566     my $storecfg = PVE::Storage::config();
2567
2568     my $list = vzlist();
2569     my $defaults = load_defaults();
2570
2571     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2572
2573     my $cpucount = $cpuinfo->{cpus} || 1;
2574
2575     foreach my $vmid (keys %$list) {
2576         next if $opt_vmid && ($vmid ne $opt_vmid);
2577
2578         my $conf = PVE::QemuConfig->load_config($vmid);
2579
2580         my $d = { vmid => $vmid };
2581         $d->{pid} = $list->{$vmid}->{pid};
2582
2583         # fixme: better status?
2584         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2585
2586         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2587         if (defined($size)) {
2588             $d->{disk} = 0; # no info available
2589             $d->{maxdisk} = $size;
2590         } else {
2591             $d->{disk} = 0;
2592             $d->{maxdisk} = 0;
2593         }
2594
2595         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2596             * ($conf->{cores} || $defaults->{cores});
2597         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2598         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2599
2600         $d->{name} = $conf->{name} || "VM $vmid";
2601         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2602             : $defaults->{memory}*(1024*1024);
2603
2604         if ($conf->{balloon}) {
2605             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2606             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2607                 : $defaults->{shares};
2608         }
2609
2610         $d->{uptime} = 0;
2611         $d->{cpu} = 0;
2612         $d->{mem} = 0;
2613
2614         $d->{netout} = 0;
2615         $d->{netin} = 0;
2616
2617         $d->{diskread} = 0;
2618         $d->{diskwrite} = 0;
2619
2620         $d->{template} = PVE::QemuConfig->is_template($conf);
2621
2622         $d->{serial} = 1 if conf_has_serial($conf);
2623         $d->{lock} = $conf->{lock} if $conf->{lock};
2624         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2625
2626         $res->{$vmid} = $d;
2627     }
2628
2629     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2630     foreach my $dev (keys %$netdev) {
2631         next if $dev !~ m/^tap([1-9]\d*)i/;
2632         my $vmid = $1;
2633         my $d = $res->{$vmid};
2634         next if !$d;
2635
2636         $d->{netout} += $netdev->{$dev}->{receive};
2637         $d->{netin} += $netdev->{$dev}->{transmit};
2638
2639         if ($full) {
2640             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2641             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2642         }
2643
2644     }
2645
2646     my $ctime = gettimeofday;
2647
2648     foreach my $vmid (keys %$list) {
2649
2650         my $d = $res->{$vmid};
2651         my $pid = $d->{pid};
2652         next if !$pid;
2653
2654         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2655         next if !$pstat; # not running
2656
2657         my $used = $pstat->{utime} + $pstat->{stime};
2658
2659         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2660
2661         if ($pstat->{vsize}) {
2662             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2663         }
2664
2665         my $old = $last_proc_pid_stat->{$pid};
2666         if (!$old) {
2667             $last_proc_pid_stat->{$pid} = {
2668                 time => $ctime,
2669                 used => $used,
2670                 cpu => 0,
2671             };
2672             next;
2673         }
2674
2675         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2676
2677         if ($dtime > 1000) {
2678             my $dutime = $used -  $old->{used};
2679
2680             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2681             $last_proc_pid_stat->{$pid} = {
2682                 time => $ctime,
2683                 used => $used,
2684                 cpu => $d->{cpu},
2685             };
2686         } else {
2687             $d->{cpu} = $old->{cpu};
2688         }
2689     }
2690
2691     return $res if !$full;
2692
2693     my $qmpclient = PVE::QMPClient->new();
2694
2695     my $ballooncb = sub {
2696         my ($vmid, $resp) = @_;
2697
2698         my $info = $resp->{'return'};
2699         return if !$info->{max_mem};
2700
2701         my $d = $res->{$vmid};
2702
2703         # use memory assigned to VM
2704         $d->{maxmem} = $info->{max_mem};
2705         $d->{balloon} = $info->{actual};
2706
2707         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2708             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2709             $d->{freemem} = $info->{free_mem};
2710         }
2711
2712         $d->{ballooninfo} = $info;
2713     };
2714
2715     my $blockstatscb = sub {
2716         my ($vmid, $resp) = @_;
2717         my $data = $resp->{'return'} || [];
2718         my $totalrdbytes = 0;
2719         my $totalwrbytes = 0;
2720
2721         for my $blockstat (@$data) {
2722             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2723             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2724
2725             $blockstat->{device} =~ s/drive-//;
2726             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2727         }
2728         $res->{$vmid}->{diskread} = $totalrdbytes;
2729         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2730     };
2731
2732     my $statuscb = sub {
2733         my ($vmid, $resp) = @_;
2734
2735         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2736         # this fails if ballon driver is not loaded, so this must be
2737         # the last commnand (following command are aborted if this fails).
2738         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2739
2740         my $status = 'unknown';
2741         if (!defined($status = $resp->{'return'}->{status})) {
2742             warn "unable to get VM status\n";
2743             return;
2744         }
2745
2746         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2747     };
2748
2749     foreach my $vmid (keys %$list) {
2750         next if $opt_vmid && ($vmid ne $opt_vmid);
2751         next if !$res->{$vmid}->{pid}; # not running
2752         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2753     }
2754
2755     $qmpclient->queue_execute(undef, 2);
2756
2757     foreach my $vmid (keys %$list) {
2758         next if $opt_vmid && ($vmid ne $opt_vmid);
2759         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2760     }
2761
2762     return $res;
2763 }
2764
2765 sub conf_has_serial {
2766     my ($conf) = @_;
2767
2768     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2769         if ($conf->{"serial$i"}) {
2770             return 1;
2771         }
2772     }
2773
2774     return 0;
2775 }
2776
2777 sub conf_has_audio {
2778     my ($conf, $id) = @_;
2779
2780     $id //= 0;
2781     my $audio = $conf->{"audio$id"};
2782     return if !defined($audio);
2783
2784     my $audioproperties = parse_property_string($audio_fmt, $audio);
2785     my $audiodriver = $audioproperties->{driver} // 'spice';
2786
2787     return {
2788         dev => $audioproperties->{device},
2789         dev_id => "audiodev$id",
2790         backend => $audiodriver,
2791         backend_id => "$audiodriver-backend${id}",
2792     };
2793 }
2794
2795 sub audio_devs {
2796     my ($audio, $audiopciaddr, $machine_version) = @_;
2797
2798     my $devs = [];
2799
2800     my $id = $audio->{dev_id};
2801     my $audiodev = "";
2802     if (min_version($machine_version, 4, 2)) {
2803         $audiodev = ",audiodev=$audio->{backend_id}";
2804     }
2805
2806     if ($audio->{dev} eq 'AC97') {
2807         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
2808     } elsif ($audio->{dev} =~ /intel\-hda$/) {
2809         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
2810         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
2811         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
2812     } else {
2813         die "unkown audio device '$audio->{dev}', implement me!";
2814     }
2815
2816     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
2817
2818     return $devs;
2819 }
2820
2821 sub vga_conf_has_spice {
2822     my ($vga) = @_;
2823
2824     my $vgaconf = parse_vga($vga);
2825     my $vgatype = $vgaconf->{type};
2826     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
2827
2828     return $1 || 1;
2829 }
2830
2831 sub is_native($) {
2832     my ($arch) = @_;
2833     return get_host_arch() eq $arch;
2834 }
2835
2836 sub get_vm_arch {
2837     my ($conf) = @_;
2838     return $conf->{arch} // get_host_arch();
2839 }
2840
2841 my $default_machines = {
2842     x86_64 => 'pc',
2843     aarch64 => 'virt',
2844 };
2845
2846 sub get_vm_machine {
2847     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
2848
2849     my $machine = $forcemachine || $conf->{machine};
2850
2851     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
2852         $arch //= 'x86_64';
2853         $machine ||= $default_machines->{$arch};
2854         if ($add_pve_version) {
2855             $kvmversion //= kvm_user_version();
2856             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
2857             $machine .= "+pve$pvever";
2858         }
2859     }
2860
2861     if ($add_pve_version && $machine !~ m/\+pve\d+$/) {
2862         # for version-pinned machines that do not include a pve-version (e.g.
2863         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
2864         $machine .= '+pve0';
2865     }
2866
2867     return $machine;
2868 }
2869
2870 sub get_ovmf_files($) {
2871     my ($arch) = @_;
2872
2873     my $ovmf = $OVMF->{$arch}
2874         or die "no OVMF images known for architecture '$arch'\n";
2875
2876     return @$ovmf;
2877 }
2878
2879 my $Arch2Qemu = {
2880     aarch64 => '/usr/bin/qemu-system-aarch64',
2881     x86_64 => '/usr/bin/qemu-system-x86_64',
2882 };
2883 sub get_command_for_arch($) {
2884     my ($arch) = @_;
2885     return '/usr/bin/kvm' if is_native($arch);
2886
2887     my $cmd = $Arch2Qemu->{$arch}
2888         or die "don't know how to emulate architecture '$arch'\n";
2889     return $cmd;
2890 }
2891
2892 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
2893 # to use in a QEMU command line (-cpu element), first array_intersect the result
2894 # of query_supported_ with query_understood_. This is necessary because:
2895 #
2896 # a) query_understood_ returns flags the host cannot use and
2897 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
2898 #    flags, but CPU settings - with most of them being flags. Those settings
2899 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
2900 #
2901 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
2902 # expensive. If you need the value returned from this, you can get it much
2903 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
2904 # $accel being 'kvm' or 'tcg'.
2905 #
2906 # pvestatd calls this function on startup and whenever the QEMU/KVM version
2907 # changes, automatically populating pmxcfs.
2908 #
2909 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
2910 # since kvm and tcg machines support different flags
2911 #
2912 sub query_supported_cpu_flags {
2913     my ($arch) = @_;
2914
2915     $arch //= get_host_arch();
2916     my $default_machine = $default_machines->{$arch};
2917
2918     my $flags = {};
2919
2920     # FIXME: Once this is merged, the code below should work for ARM as well:
2921     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
2922     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
2923         $arch eq "aarch64";
2924
2925     my $kvm_supported = defined(kvm_version());
2926     my $qemu_cmd = get_command_for_arch($arch);
2927     my $fakevmid = -1;
2928     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
2929
2930     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
2931     my $query_supported_run_qemu = sub {
2932         my ($kvm) = @_;
2933
2934         my $flags = {};
2935         my $cmd = [
2936             $qemu_cmd,
2937             '-machine', $default_machine,
2938             '-display', 'none',
2939             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server,nowait",
2940             '-mon', 'chardev=qmp,mode=control',
2941             '-pidfile', $pidfile,
2942             '-S', '-daemonize'
2943         ];
2944
2945         if (!$kvm) {
2946             push @$cmd, '-accel', 'tcg';
2947         }
2948
2949         my $rc = run_command($cmd, noerr => 1, quiet => 0);
2950         die "QEMU flag querying VM exited with code " . $rc if $rc;
2951
2952         eval {
2953             my $cmd_result = mon_cmd(
2954                 $fakevmid,
2955                 'query-cpu-model-expansion',
2956                 type => 'full',
2957                 model => { name => 'host' }
2958             );
2959
2960             my $props = $cmd_result->{model}->{props};
2961             foreach my $prop (keys %$props) {
2962                 next if $props->{$prop} ne '1';
2963                 # QEMU returns some flags multiple times, with '_', '.' or '-'
2964                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
2965                 # We only keep those with underscores, to match /proc/cpuinfo
2966                 $prop =~ s/\.|-/_/g;
2967                 $flags->{$prop} = 1;
2968             }
2969         };
2970         my $err = $@;
2971
2972         # force stop with 10 sec timeout and 'nocheck'
2973         # always stop, even if QMP failed
2974         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
2975
2976         die $err if $err;
2977
2978         return [ sort keys %$flags ];
2979     };
2980
2981     # We need to query QEMU twice, since KVM and TCG have different supported flags
2982     PVE::QemuConfig->lock_config($fakevmid, sub {
2983         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
2984         warn "warning: failed querying supported tcg flags: $@\n" if $@;
2985
2986         if ($kvm_supported) {
2987             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
2988             warn "warning: failed querying supported kvm flags: $@\n" if $@;
2989         }
2990     });
2991
2992     return $flags;
2993 }
2994
2995 # Understood CPU flags are written to a file at 'pve-qemu' compile time
2996 my $understood_cpu_flag_dir = "/usr/share/kvm";
2997 sub query_understood_cpu_flags {
2998     my $arch = get_host_arch();
2999     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3000
3001     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3002         if ! -e $filepath;
3003
3004     my $raw = file_get_contents($filepath);
3005     $raw =~ s/^\s+|\s+$//g;
3006     my @flags = split(/\s+/, $raw);
3007
3008     return \@flags;
3009 }
3010
3011 sub config_to_command {
3012     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu) = @_;
3013
3014     my $cmd = [];
3015     my $globalFlags = [];
3016     my $machineFlags = [];
3017     my $rtcFlags = [];
3018     my $devices = [];
3019     my $pciaddr = '';
3020     my $bridges = {};
3021     my $ostype = $conf->{ostype};
3022     my $winversion = windows_version($ostype);
3023     my $kvm = $conf->{kvm};
3024     my $nodename = nodename();
3025
3026     my $arch = get_vm_arch($conf);
3027     my $kvm_binary = get_command_for_arch($arch);
3028     my $kvmver = kvm_user_version($kvm_binary);
3029
3030     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3031         $kvmver //= "undefined";
3032         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3033     }
3034
3035     my $add_pve_version = min_version($kvmver, 4, 1);
3036
3037     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3038     my $machine_version = extract_version($machine_type, $kvmver);
3039     $kvm //= 1 if is_native($arch);
3040
3041     $machine_version =~ m/(\d+)\.(\d+)/;
3042     my ($machine_major, $machine_minor) = ($1, $2);
3043
3044     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3045         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3046     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3047         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3048             ." please upgrade node '$nodename'\n"
3049     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3050         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3051         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3052             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3053             ." node '$nodename'\n";
3054     }
3055
3056     # if a specific +pve version is required for a feature, use $version_guard
3057     # instead of min_version to allow machines to be run with the minimum
3058     # required version
3059     my $required_pve_version = 0;
3060     my $version_guard = sub {
3061         my ($major, $minor, $pve) = @_;
3062         return 0 if !min_version($machine_version, $major, $minor, $pve);
3063         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3064         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3065         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3066         return 1;
3067     };
3068
3069     if ($kvm && !defined kvm_version()) {
3070         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3071             ." or enable in BIOS.\n";
3072     }
3073
3074     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3075     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3076     my $use_old_bios_files = undef;
3077     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3078
3079     my $cpuunits = defined($conf->{cpuunits}) ?
3080             $conf->{cpuunits} : $defaults->{cpuunits};
3081
3082     push @$cmd, $kvm_binary;
3083
3084     push @$cmd, '-id', $vmid;
3085
3086     my $vmname = $conf->{name} || "vm$vmid";
3087
3088     push @$cmd, '-name', $vmname;
3089
3090     my $use_virtio = 0;
3091
3092     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3093     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3094     push @$cmd, '-mon', "chardev=qmp,mode=control";
3095
3096     if (min_version($machine_version, 2, 12)) {
3097         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3098         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3099     }
3100
3101     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3102
3103     push @$cmd, '-daemonize';
3104
3105     if ($conf->{smbios1}) {
3106         my $smbios_conf = parse_smbios1($conf->{smbios1});
3107         if ($smbios_conf->{base64}) {
3108             # Do not pass base64 flag to qemu
3109             delete $smbios_conf->{base64};
3110             my $smbios_string = "";
3111             foreach my $key (keys %$smbios_conf) {
3112                 my $value;
3113                 if ($key eq "uuid") {
3114                     $value = $smbios_conf->{uuid}
3115                 } else {
3116                     $value = decode_base64($smbios_conf->{$key});
3117                 }
3118                 # qemu accepts any binary data, only commas need escaping by double comma
3119                 $value =~ s/,/,,/g;
3120                 $smbios_string .= "," . $key . "=" . $value if $value;
3121             }
3122             push @$cmd, '-smbios', "type=1" . $smbios_string;
3123         } else {
3124             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3125         }
3126     }
3127
3128     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3129         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3130         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3131
3132         my ($path, $format);
3133         if (my $efidisk = $conf->{efidisk0}) {
3134             my $d = parse_drive('efidisk0', $efidisk);
3135             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3136             $format = $d->{format};
3137             if ($storeid) {
3138                 $path = PVE::Storage::path($storecfg, $d->{file});
3139                 if (!defined($format)) {
3140                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3141                     $format = qemu_img_format($scfg, $volname);
3142                 }
3143             } else {
3144                 $path = $d->{file};
3145                 die "efidisk format must be specified\n"
3146                     if !defined($format);
3147             }
3148         } else {
3149             warn "no efidisk configured! Using temporary efivars disk.\n";
3150             $path = "/tmp/$vmid-ovmf.fd";
3151             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3152             $format = 'raw';
3153         }
3154
3155         my $size_str = "";
3156
3157         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3158             $size_str = ",size=" . (-s $ovmf_vars);
3159         }
3160
3161         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3162         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0$size_str,file=$path";
3163     }
3164
3165     # load q35 config
3166     if ($q35) {
3167         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3168         if (min_version($machine_version, 4, 0)) {
3169             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3170         } else {
3171             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3172         }
3173     }
3174
3175     if ($conf->{vmgenid}) {
3176         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3177     }
3178
3179     # add usb controllers
3180     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3181         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3182     push @$devices, @usbcontrollers if @usbcontrollers;
3183     my $vga = parse_vga($conf->{vga});
3184
3185     my $qxlnum = vga_conf_has_spice($conf->{vga});
3186     $vga->{type} = 'qxl' if $qxlnum;
3187
3188     if (!$vga->{type}) {
3189         if ($arch eq 'aarch64') {
3190             $vga->{type} = 'virtio';
3191         } elsif (min_version($machine_version, 2, 9)) {
3192             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3193         } else {
3194             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3195         }
3196     }
3197
3198     # enable absolute mouse coordinates (needed by vnc)
3199     my $tablet;
3200     if (defined($conf->{tablet})) {
3201         $tablet = $conf->{tablet};
3202     } else {
3203         $tablet = $defaults->{tablet};
3204         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3205         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3206     }
3207
3208     if ($tablet) {
3209         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3210         my $kbd = print_keyboarddevice_full($conf, $arch);
3211         push @$devices, '-device', $kbd if defined($kbd);
3212     }
3213
3214     my $bootorder = device_bootorder($conf);
3215
3216     # host pci device passthrough
3217     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3218         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3219
3220     # usb devices
3221     my $usb_dev_features = {};
3222     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3223
3224     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3225         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3226     push @$devices, @usbdevices if @usbdevices;
3227
3228     # serial devices
3229     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3230         if (my $path = $conf->{"serial$i"}) {
3231             if ($path eq 'socket') {
3232                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3233                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3234                 # On aarch64, serial0 is the UART device. Qemu only allows
3235                 # connecting UART devices via the '-serial' command line, as
3236                 # the device has a fixed slot on the hardware...
3237                 if ($arch eq 'aarch64' && $i == 0) {
3238                     push @$devices, '-serial', "chardev:serial$i";
3239                 } else {
3240                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3241                 }
3242             } else {
3243                 die "no such serial device\n" if ! -c $path;
3244                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3245                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3246             }
3247         }
3248     }
3249
3250     # parallel devices
3251     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3252         if (my $path = $conf->{"parallel$i"}) {
3253             die "no such parallel device\n" if ! -c $path;
3254             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3255             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3256             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3257         }
3258     }
3259
3260     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3261         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3262         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3263         push @$devices, @$audio_devs;
3264     }
3265
3266     my $sockets = 1;
3267     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3268     $sockets = $conf->{sockets} if  $conf->{sockets};
3269
3270     my $cores = $conf->{cores} || 1;
3271
3272     my $maxcpus = $sockets * $cores;
3273
3274     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3275
3276     my $allowed_vcpus = $cpuinfo->{cpus};
3277
3278     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3279         if ($allowed_vcpus < $maxcpus);
3280
3281     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3282
3283         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3284         for (my $i = 2; $i <= $vcpus; $i++)  {
3285             my $cpustr = print_cpu_device($conf,$i);
3286             push @$cmd, '-device', $cpustr;
3287         }
3288
3289     } else {
3290
3291         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3292     }
3293     push @$cmd, '-nodefaults';
3294
3295     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3296
3297     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3298
3299     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3300
3301     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3302         push @$devices, '-device', print_vga_device(
3303             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3304         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3305         push @$cmd,  '-vnc', "unix:$socket,password";
3306     } else {
3307         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3308         push @$cmd, '-nographic';
3309     }
3310
3311     # time drift fix
3312     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3313     my $useLocaltime = $conf->{localtime};
3314
3315     if ($winversion >= 5) { # windows
3316         $useLocaltime = 1 if !defined($conf->{localtime});
3317
3318         # use time drift fix when acpi is enabled
3319         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3320             $tdf = 1 if !defined($conf->{tdf});
3321         }
3322     }
3323
3324     if ($winversion >= 6) {
3325         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3326         push @$cmd, '-no-hpet';
3327     }
3328
3329     push @$rtcFlags, 'driftfix=slew' if $tdf;
3330
3331     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3332         push @$rtcFlags, "base=$conf->{startdate}";
3333     } elsif ($useLocaltime) {
3334         push @$rtcFlags, 'base=localtime';
3335     }
3336
3337     if ($forcecpu) {
3338         push @$cmd, '-cpu', $forcecpu;
3339     } else {
3340         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3341     }
3342
3343     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3344
3345     push @$cmd, '-S' if $conf->{freeze};
3346
3347     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3348
3349     my $guest_agent = parse_guest_agent($conf);
3350
3351     if ($guest_agent->{enabled}) {
3352         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3353         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3354
3355         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3356             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3357             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3358             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3359         } elsif ($guest_agent->{type} eq 'isa') {
3360             push @$devices, '-device', "isa-serial,chardev=qga0";
3361         }
3362     }
3363
3364     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3365     if ($rng && $version_guard->(4, 1, 2)) {
3366         check_rng_source($rng->{source});
3367
3368         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3369         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3370         my $limiter_str = "";
3371         if ($max_bytes) {
3372             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3373         }
3374
3375         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3376         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3377         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3378     }
3379
3380     my $spice_port;
3381
3382     if ($qxlnum) {
3383         if ($qxlnum > 1) {
3384             if ($winversion){
3385                 for (my $i = 1; $i < $qxlnum; $i++){
3386                     push @$devices, '-device', print_vga_device(
3387                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3388                 }
3389             } else {
3390                 # assume other OS works like Linux
3391                 my ($ram, $vram) = ("134217728", "67108864");
3392                 if ($vga->{memory}) {
3393                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3394                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3395                 }
3396                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3397                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3398             }
3399         }
3400
3401         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3402
3403         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3404         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3405         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3406
3407         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3408         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3409         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3410
3411         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3412         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3413
3414         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3415         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3416         if ($spice_enhancement->{foldersharing}) {
3417             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3418             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3419         }
3420
3421         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3422         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3423             if $spice_enhancement->{videostreaming};
3424
3425         push @$devices, '-spice', "$spice_opts";
3426     }
3427
3428     # enable balloon by default, unless explicitly disabled
3429     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3430         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3431         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3432     }
3433
3434     if ($conf->{watchdog}) {
3435         my $wdopts = parse_watchdog($conf->{watchdog});
3436         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3437         my $watchdog = $wdopts->{model} || 'i6300esb';
3438         push @$devices, '-device', "$watchdog$pciaddr";
3439         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3440     }
3441
3442     my $vollist = [];
3443     my $scsicontroller = {};
3444     my $ahcicontroller = {};
3445     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3446
3447     # Add iscsi initiator name if available
3448     if (my $initiator = get_initiator_name()) {
3449         push @$devices, '-iscsi', "initiator-name=$initiator";
3450     }
3451
3452     PVE::QemuConfig->foreach_volume($conf, sub {
3453         my ($ds, $drive) = @_;
3454
3455         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3456             push @$vollist, $drive->{file};
3457         }
3458
3459         # ignore efidisk here, already added in bios/fw handling code above
3460         return if $drive->{interface} eq 'efidisk';
3461
3462         $use_virtio = 1 if $ds =~ m/^virtio/;
3463
3464         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3465
3466         if ($drive->{interface} eq 'virtio'){
3467            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3468         }
3469
3470         if ($drive->{interface} eq 'scsi') {
3471
3472             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3473
3474             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3475                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3476
3477             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3478             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3479
3480             my $iothread = '';
3481             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3482                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3483                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3484             } elsif ($drive->{iothread}) {
3485                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3486             }
3487
3488             my $queues = '';
3489             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3490                 $queues = ",num_queues=$drive->{queues}";
3491             }
3492
3493             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3494                 if !$scsicontroller->{$controller};
3495             $scsicontroller->{$controller}=1;
3496         }
3497
3498         if ($drive->{interface} eq 'sata') {
3499             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3500             $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3501             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3502                 if !$ahcicontroller->{$controller};
3503             $ahcicontroller->{$controller}=1;
3504         }
3505
3506         my $drive_cmd = print_drive_commandline_full($storecfg, $vmid, $drive);
3507         $drive_cmd .= ',readonly' if PVE::QemuConfig->is_template($conf);
3508
3509         push @$devices, '-drive',$drive_cmd;
3510         push @$devices, '-device', print_drivedevice_full(
3511             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3512     });
3513
3514     for (my $i = 0; $i < $MAX_NETS; $i++) {
3515         my $netname = "net$i";
3516
3517         next if !$conf->{$netname};
3518         my $d = parse_net($conf->{$netname});
3519         next if !$d;
3520
3521         $use_virtio = 1 if $d->{model} eq 'virtio';
3522
3523         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
3524
3525         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
3526         push @$devices, '-netdev', $netdevfull;
3527
3528         my $netdevicefull = print_netdevice_full(
3529             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
3530
3531         push @$devices, '-device', $netdevicefull;
3532     }
3533
3534     if ($conf->{ivshmem}) {
3535         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3536
3537         my $bus;
3538         if ($q35) {
3539             $bus = print_pcie_addr("ivshmem");
3540         } else {
3541             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3542         }
3543
3544         my $ivshmem_name = $ivshmem->{name} // $vmid;
3545         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3546
3547         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3548         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
3549             .",size=$ivshmem->{size}M";
3550     }
3551
3552     # pci.4 is nested in pci.1
3553     $bridges->{1} = 1 if $bridges->{4};
3554
3555     if (!$q35) {
3556         # add pci bridges
3557         if (min_version($machine_version, 2, 3)) {
3558            $bridges->{1} = 1;
3559            $bridges->{2} = 1;
3560         }
3561
3562         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3563
3564     }
3565
3566     for my $k (sort {$b cmp $a} keys %$bridges) {
3567         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3568
3569         my $k_name = $k;
3570         if ($k == 2 && $legacy_igd) {
3571             $k_name = "$k-igd";
3572         }
3573         $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
3574
3575         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3576         if ($q35) {
3577             # add after -readconfig pve-q35.cfg
3578             splice @$devices, 2, 0, '-device', $devstr;
3579         } else {
3580             unshift @$devices, '-device', $devstr if $k > 0;
3581         }
3582     }
3583
3584     if (!$kvm) {
3585         push @$machineFlags, 'accel=tcg';
3586     }
3587
3588     my $machine_type_min = $machine_type;
3589     if ($add_pve_version) {
3590         $machine_type_min =~ s/\+pve\d+$//;
3591         $machine_type_min .= "+pve$required_pve_version";
3592     }
3593     push @$machineFlags, "type=${machine_type_min}";
3594
3595     push @$cmd, @$devices;
3596     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
3597     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
3598     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
3599
3600     if (my $vmstate = $conf->{vmstate}) {
3601         my $statepath = PVE::Storage::path($storecfg, $vmstate);
3602         push @$vollist, $vmstate;
3603         push @$cmd, '-loadstate', $statepath;
3604         print "activating and using '$vmstate' as vmstate\n";
3605     }
3606
3607     # add custom args
3608     if ($conf->{args}) {
3609         my $aa = PVE::Tools::split_args($conf->{args});
3610         push @$cmd, @$aa;
3611     }
3612
3613     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3614 }
3615
3616 sub check_rng_source {
3617     my ($source) = @_;
3618
3619     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3620     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
3621         if ! -e $source;
3622
3623     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
3624     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
3625         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
3626         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
3627         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
3628             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
3629             ." to the host.\n";
3630     }
3631 }
3632
3633 sub spice_port {
3634     my ($vmid) = @_;
3635
3636     my $res = mon_cmd($vmid, 'query-spice');
3637
3638     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3639 }
3640
3641 sub vm_devices_list {
3642     my ($vmid) = @_;
3643
3644     my $res = mon_cmd($vmid, 'query-pci');
3645     my $devices_to_check = [];
3646     my $devices = {};
3647     foreach my $pcibus (@$res) {
3648         push @$devices_to_check, @{$pcibus->{devices}},
3649     }
3650
3651     while (@$devices_to_check) {
3652         my $to_check = [];
3653         for my $d (@$devices_to_check) {
3654             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3655             next if !$d->{'pci_bridge'};
3656
3657             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3658             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3659         }
3660         $devices_to_check = $to_check;
3661     }
3662
3663     my $resblock = mon_cmd($vmid, 'query-block');
3664     foreach my $block (@$resblock) {
3665         if($block->{device} =~ m/^drive-(\S+)/){
3666                 $devices->{$1} = 1;
3667         }
3668     }
3669
3670     my $resmice = mon_cmd($vmid, 'query-mice');
3671     foreach my $mice (@$resmice) {
3672         if ($mice->{name} eq 'QEMU HID Tablet') {
3673             $devices->{tablet} = 1;
3674             last;
3675         }
3676     }
3677
3678     # for usb devices there is no query-usb
3679     # but we can iterate over the entries in
3680     # qom-list path=/machine/peripheral
3681     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3682     foreach my $per (@$resperipheral) {
3683         if ($per->{name} =~ m/^usb\d+$/) {
3684             $devices->{$per->{name}} = 1;
3685         }
3686     }
3687
3688     return $devices;
3689 }
3690
3691 sub vm_deviceplug {
3692     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3693
3694     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3695
3696     my $devices_list = vm_devices_list($vmid);
3697     return 1 if defined($devices_list->{$deviceid});
3698
3699     # add PCI bridge if we need it for the device
3700     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
3701
3702     if ($deviceid eq 'tablet') {
3703
3704         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
3705
3706     } elsif ($deviceid eq 'keyboard') {
3707
3708         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
3709
3710     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3711
3712         die "usb hotplug currently not reliable\n";
3713         # since we can't reliably hot unplug all added usb devices and usb
3714         # passthrough breaks live migration we disable usb hotplugging for now
3715         #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3716
3717     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3718
3719         qemu_iothread_add($vmid, $deviceid, $device);
3720
3721         qemu_driveadd($storecfg, $vmid, $device);
3722         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
3723
3724         qemu_deviceadd($vmid, $devicefull);
3725         eval { qemu_deviceaddverify($vmid, $deviceid); };
3726         if (my $err = $@) {
3727             eval { qemu_drivedel($vmid, $deviceid); };
3728             warn $@ if $@;
3729             die $err;
3730         }
3731
3732     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3733
3734
3735         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3736         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3737         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3738
3739         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3740
3741         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3742             qemu_iothread_add($vmid, $deviceid, $device);
3743             $devicefull .= ",iothread=iothread-$deviceid";
3744         }
3745
3746         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3747             $devicefull .= ",num_queues=$device->{queues}";
3748         }
3749
3750         qemu_deviceadd($vmid, $devicefull);
3751         qemu_deviceaddverify($vmid, $deviceid);
3752
3753     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3754
3755         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
3756         qemu_driveadd($storecfg, $vmid, $device);
3757
3758         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
3759         eval { qemu_deviceadd($vmid, $devicefull); };
3760         if (my $err = $@) {
3761             eval { qemu_drivedel($vmid, $deviceid); };
3762             warn $@ if $@;
3763             die $err;
3764         }
3765
3766     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3767
3768         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
3769
3770         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
3771         my $use_old_bios_files = undef;
3772         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3773
3774         my $netdevicefull = print_netdevice_full(
3775             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
3776         qemu_deviceadd($vmid, $netdevicefull);
3777         eval {
3778             qemu_deviceaddverify($vmid, $deviceid);
3779             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
3780         };
3781         if (my $err = $@) {
3782             eval { qemu_netdevdel($vmid, $deviceid); };
3783             warn $@ if $@;
3784             die $err;
3785         }
3786
3787     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3788
3789         my $bridgeid = $2;
3790         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3791         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3792
3793         qemu_deviceadd($vmid, $devicefull);
3794         qemu_deviceaddverify($vmid, $deviceid);
3795
3796     } else {
3797         die "can't hotplug device '$deviceid'\n";
3798     }
3799
3800     return 1;
3801 }
3802
3803 # fixme: this should raise exceptions on error!
3804 sub vm_deviceunplug {
3805     my ($vmid, $conf, $deviceid) = @_;
3806
3807     my $devices_list = vm_devices_list($vmid);
3808     return 1 if !defined($devices_list->{$deviceid});
3809
3810     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
3811     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
3812
3813     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
3814
3815         qemu_devicedel($vmid, $deviceid);
3816
3817     } elsif ($deviceid =~ m/^usb\d+$/) {
3818
3819         die "usb hotplug currently not reliable\n";
3820         # when unplugging usb devices this way, there may be remaining usb
3821         # controllers/hubs so we disable it for now
3822         #qemu_devicedel($vmid, $deviceid);
3823         #qemu_devicedelverify($vmid, $deviceid);
3824
3825     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3826
3827         qemu_devicedel($vmid, $deviceid);
3828         qemu_devicedelverify($vmid, $deviceid);
3829         qemu_drivedel($vmid, $deviceid);
3830         qemu_iothread_del($conf, $vmid, $deviceid);
3831
3832     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3833
3834         qemu_devicedel($vmid, $deviceid);
3835         qemu_devicedelverify($vmid, $deviceid);
3836         qemu_iothread_del($conf, $vmid, $deviceid);
3837
3838     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3839
3840         qemu_devicedel($vmid, $deviceid);
3841         qemu_drivedel($vmid, $deviceid);
3842         qemu_deletescsihw($conf, $vmid, $deviceid);
3843
3844     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3845
3846         qemu_devicedel($vmid, $deviceid);
3847         qemu_devicedelverify($vmid, $deviceid);
3848         qemu_netdevdel($vmid, $deviceid);
3849
3850     } else {
3851         die "can't unplug device '$deviceid'\n";
3852     }
3853
3854     return 1;
3855 }
3856
3857 sub qemu_deviceadd {
3858     my ($vmid, $devicefull) = @_;
3859
3860     $devicefull = "driver=".$devicefull;
3861     my %options =  split(/[=,]/, $devicefull);
3862
3863     mon_cmd($vmid, "device_add" , %options);
3864 }
3865
3866 sub qemu_devicedel {
3867     my ($vmid, $deviceid) = @_;
3868
3869     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
3870 }
3871
3872 sub qemu_iothread_add {
3873     my($vmid, $deviceid, $device) = @_;
3874
3875     if ($device->{iothread}) {
3876         my $iothreads = vm_iothreads_list($vmid);
3877         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3878     }
3879 }
3880
3881 sub qemu_iothread_del {
3882     my($conf, $vmid, $deviceid) = @_;
3883
3884     my $confid = $deviceid;
3885     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
3886         $confid = 'scsi' . $1;
3887     }
3888     my $device = parse_drive($confid, $conf->{$confid});
3889     if ($device->{iothread}) {
3890         my $iothreads = vm_iothreads_list($vmid);
3891         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3892     }
3893 }
3894
3895 sub qemu_objectadd {
3896     my($vmid, $objectid, $qomtype) = @_;
3897
3898     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3899
3900     return 1;
3901 }
3902
3903 sub qemu_objectdel {
3904     my($vmid, $objectid) = @_;
3905
3906     mon_cmd($vmid, "object-del", id => $objectid);
3907
3908     return 1;
3909 }
3910
3911 sub qemu_driveadd {
3912     my ($storecfg, $vmid, $device) = @_;
3913
3914     my $drive = print_drive_commandline_full($storecfg, $vmid, $device);
3915     $drive =~ s/\\/\\\\/g;
3916     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
3917
3918     # If the command succeeds qemu prints: "OK"
3919     return 1 if $ret =~ m/OK/s;
3920
3921     die "adding drive failed: $ret\n";
3922 }
3923
3924 sub qemu_drivedel {
3925     my($vmid, $deviceid) = @_;
3926
3927     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
3928     $ret =~ s/^\s+//;
3929
3930     return 1 if $ret eq "";
3931
3932     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3933     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3934
3935     die "deleting drive $deviceid failed : $ret\n";
3936 }
3937
3938 sub qemu_deviceaddverify {
3939     my ($vmid, $deviceid) = @_;
3940
3941     for (my $i = 0; $i <= 5; $i++) {
3942          my $devices_list = vm_devices_list($vmid);
3943          return 1 if defined($devices_list->{$deviceid});
3944          sleep 1;
3945     }
3946
3947     die "error on hotplug device '$deviceid'\n";
3948 }
3949
3950
3951 sub qemu_devicedelverify {
3952     my ($vmid, $deviceid) = @_;
3953
3954     # need to verify that the device is correctly removed as device_del
3955     # is async and empty return is not reliable
3956
3957     for (my $i = 0; $i <= 5; $i++) {
3958          my $devices_list = vm_devices_list($vmid);
3959          return 1 if !defined($devices_list->{$deviceid});
3960          sleep 1;
3961     }
3962
3963     die "error on hot-unplugging device '$deviceid'\n";
3964 }
3965
3966 sub qemu_findorcreatescsihw {
3967     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
3968
3969     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3970
3971     my $scsihwid="$controller_prefix$controller";
3972     my $devices_list = vm_devices_list($vmid);
3973
3974     if(!defined($devices_list->{$scsihwid})) {
3975         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
3976     }
3977
3978     return 1;
3979 }
3980
3981 sub qemu_deletescsihw {
3982     my ($conf, $vmid, $opt) = @_;
3983
3984     my $device = parse_drive($opt, $conf->{$opt});
3985
3986     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3987         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3988         return 1;
3989     }
3990
3991     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3992
3993     my $devices_list = vm_devices_list($vmid);
3994     foreach my $opt (keys %{$devices_list}) {
3995         if (is_valid_drivename($opt)) {
3996             my $drive = parse_drive($opt, $conf->{$opt});
3997             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3998                 return 1;
3999             }
4000         }
4001     }
4002
4003     my $scsihwid="scsihw$controller";
4004
4005     vm_deviceunplug($vmid, $conf, $scsihwid);
4006
4007     return 1;
4008 }
4009
4010 sub qemu_add_pci_bridge {
4011     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4012
4013     my $bridges = {};
4014
4015     my $bridgeid;
4016
4017     print_pci_addr($device, $bridges, $arch, $machine_type);
4018
4019     while (my ($k, $v) = each %$bridges) {
4020         $bridgeid = $k;
4021     }
4022     return 1 if !defined($bridgeid) || $bridgeid < 1;
4023
4024     my $bridge = "pci.$bridgeid";
4025     my $devices_list = vm_devices_list($vmid);
4026
4027     if (!defined($devices_list->{$bridge})) {
4028         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4029     }
4030
4031     return 1;
4032 }
4033
4034 sub qemu_set_link_status {
4035     my ($vmid, $device, $up) = @_;
4036
4037     mon_cmd($vmid, "set_link", name => $device,
4038                up => $up ? JSON::true : JSON::false);
4039 }
4040
4041 sub qemu_netdevadd {
4042     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4043
4044     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4045     my %options =  split(/[=,]/, $netdev);
4046
4047     if (defined(my $vhost = $options{vhost})) {
4048         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4049     }
4050
4051     if (defined(my $queues = $options{queues})) {
4052         $options{queues} = $queues + 0;
4053     }
4054
4055     mon_cmd($vmid, "netdev_add",  %options);
4056     return 1;
4057 }
4058
4059 sub qemu_netdevdel {
4060     my ($vmid, $deviceid) = @_;
4061
4062     mon_cmd($vmid, "netdev_del", id => $deviceid);
4063 }
4064
4065 sub qemu_usb_hotplug {
4066     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4067
4068     return if !$device;
4069
4070     # remove the old one first
4071     vm_deviceunplug($vmid, $conf, $deviceid);
4072
4073     # check if xhci controller is necessary and available
4074     if ($device->{usb3}) {
4075
4076         my $devicelist = vm_devices_list($vmid);
4077
4078         if (!$devicelist->{xhci}) {
4079             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4080             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4081         }
4082     }
4083     my $d = parse_usb_device($device->{host});
4084     $d->{usb3} = $device->{usb3};
4085
4086     # add the new one
4087     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4088 }
4089
4090 sub qemu_cpu_hotplug {
4091     my ($vmid, $conf, $vcpus) = @_;
4092
4093     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4094
4095     my $sockets = 1;
4096     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4097     $sockets = $conf->{sockets} if  $conf->{sockets};
4098     my $cores = $conf->{cores} || 1;
4099     my $maxcpus = $sockets * $cores;
4100
4101     $vcpus = $maxcpus if !$vcpus;
4102
4103     die "you can't add more vcpus than maxcpus\n"
4104         if $vcpus > $maxcpus;
4105
4106     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4107
4108     if ($vcpus < $currentvcpus) {
4109
4110         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4111
4112             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4113                 qemu_devicedel($vmid, "cpu$i");
4114                 my $retry = 0;
4115                 my $currentrunningvcpus = undef;
4116                 while (1) {
4117                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4118                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4119                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4120                     $retry++;
4121                     sleep 1;
4122                 }
4123                 #update conf after each succesfull cpu unplug
4124                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4125                 PVE::QemuConfig->write_config($vmid, $conf);
4126             }
4127         } else {
4128             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4129         }
4130
4131         return;
4132     }
4133
4134     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4135     die "vcpus in running vm does not match its configuration\n"
4136         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4137
4138     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4139
4140         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4141             my $cpustr = print_cpu_device($conf, $i);
4142             qemu_deviceadd($vmid, $cpustr);
4143
4144             my $retry = 0;
4145             my $currentrunningvcpus = undef;
4146             while (1) {
4147                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4148                 last if scalar(@{$currentrunningvcpus}) == $i;
4149                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4150                 sleep 1;
4151                 $retry++;
4152             }
4153             #update conf after each succesfull cpu hotplug
4154             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4155             PVE::QemuConfig->write_config($vmid, $conf);
4156         }
4157     } else {
4158
4159         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4160             mon_cmd($vmid, "cpu-add", id => int($i));
4161         }
4162     }
4163 }
4164
4165 sub qemu_block_set_io_throttle {
4166     my ($vmid, $deviceid,
4167         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4168         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4169         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4170         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4171
4172     return if !check_running($vmid) ;
4173
4174     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4175         bps => int($bps),
4176         bps_rd => int($bps_rd),
4177         bps_wr => int($bps_wr),
4178         iops => int($iops),
4179         iops_rd => int($iops_rd),
4180         iops_wr => int($iops_wr),
4181         bps_max => int($bps_max),
4182         bps_rd_max => int($bps_rd_max),
4183         bps_wr_max => int($bps_wr_max),
4184         iops_max => int($iops_max),
4185         iops_rd_max => int($iops_rd_max),
4186         iops_wr_max => int($iops_wr_max),
4187         bps_max_length => int($bps_max_length),
4188         bps_rd_max_length => int($bps_rd_max_length),
4189         bps_wr_max_length => int($bps_wr_max_length),
4190         iops_max_length => int($iops_max_length),
4191         iops_rd_max_length => int($iops_rd_max_length),
4192         iops_wr_max_length => int($iops_wr_max_length),
4193     );
4194
4195 }
4196
4197 sub qemu_block_resize {
4198     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4199
4200     my $running = check_running($vmid);
4201
4202     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4203
4204     return if !$running;
4205
4206     my $padding = (1024 - $size % 1024) % 1024;
4207     $size = $size + $padding;
4208
4209     mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4210
4211 }
4212
4213 sub qemu_volume_snapshot {
4214     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4215
4216     my $running = check_running($vmid);
4217
4218     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4219         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4220     } else {
4221         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4222     }
4223 }
4224
4225 sub qemu_volume_snapshot_delete {
4226     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4227
4228     my $running = check_running($vmid);
4229
4230     if($running) {
4231
4232         $running = undef;
4233         my $conf = PVE::QemuConfig->load_config($vmid);
4234         PVE::QemuConfig->foreach_volume($conf, sub {
4235             my ($ds, $drive) = @_;
4236             $running = 1 if $drive->{file} eq $volid;
4237         });
4238     }
4239
4240     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4241         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4242     } else {
4243         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4244     }
4245 }
4246
4247 sub set_migration_caps {
4248     my ($vmid) = @_;
4249
4250     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4251
4252     my $cap_ref = [];
4253
4254     my $enabled_cap = {
4255         "auto-converge" => 1,
4256         "xbzrle" => 1,
4257         "x-rdma-pin-all" => 0,
4258         "zero-blocks" => 0,
4259         "compress" => 0,
4260         "dirty-bitmaps" => $qemu_support->{'pbs-dirty-bitmap-migration'} ? 1 : 0,
4261     };
4262
4263     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4264
4265     for my $supported_capability (@$supported_capabilities) {
4266         push @$cap_ref, {
4267             capability => $supported_capability->{capability},
4268             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4269         };
4270     }
4271
4272     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4273 }
4274
4275 sub foreach_volid {
4276     my ($conf, $func, @param) = @_;
4277
4278     my $volhash = {};
4279
4280     my $test_volid = sub {
4281         my ($key, $drive, $snapname) = @_;
4282
4283         my $volid = $drive->{file};
4284         return if !$volid;
4285
4286         $volhash->{$volid}->{cdrom} //= 1;
4287         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4288
4289         my $replicate = $drive->{replicate} // 1;
4290         $volhash->{$volid}->{replicate} //= 0;
4291         $volhash->{$volid}->{replicate} = 1 if $replicate;
4292
4293         $volhash->{$volid}->{shared} //= 0;
4294         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4295
4296         $volhash->{$volid}->{referenced_in_config} //= 0;
4297         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4298
4299         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4300             if defined($snapname);
4301
4302         my $size = $drive->{size};
4303         $volhash->{$volid}->{size} //= $size if $size;
4304
4305         $volhash->{$volid}->{is_vmstate} //= 0;
4306         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4307
4308         $volhash->{$volid}->{is_unused} //= 0;
4309         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4310     };
4311
4312     my $include_opts = {
4313         extra_keys => ['vmstate'],
4314         include_unused => 1,
4315     };
4316
4317     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4318     foreach my $snapname (keys %{$conf->{snapshots}}) {
4319         my $snap = $conf->{snapshots}->{$snapname};
4320         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4321     }
4322
4323     foreach my $volid (keys %$volhash) {
4324         &$func($volid, $volhash->{$volid}, @param);
4325     }
4326 }
4327
4328 my $fast_plug_option = {
4329     'lock' => 1,
4330     'name' => 1,
4331     'onboot' => 1,
4332     'shares' => 1,
4333     'startup' => 1,
4334     'description' => 1,
4335     'protection' => 1,
4336     'vmstatestorage' => 1,
4337     'hookscript' => 1,
4338     'tags' => 1,
4339 };
4340
4341 # hotplug changes in [PENDING]
4342 # $selection hash can be used to only apply specified options, for
4343 # example: { cores => 1 } (only apply changed 'cores')
4344 # $errors ref is used to return error messages
4345 sub vmconfig_hotplug_pending {
4346     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4347
4348     my $defaults = load_defaults();
4349     my $arch = get_vm_arch($conf);
4350     my $machine_type = get_vm_machine($conf, undef, $arch);
4351
4352     # commit values which do not have any impact on running VM first
4353     # Note: those option cannot raise errors, we we do not care about
4354     # $selection and always apply them.
4355
4356     my $add_error = sub {
4357         my ($opt, $msg) = @_;
4358         $errors->{$opt} = "hotplug problem - $msg";
4359     };
4360
4361     my $changes = 0;
4362     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4363         if ($fast_plug_option->{$opt}) {
4364             $conf->{$opt} = $conf->{pending}->{$opt};
4365             delete $conf->{pending}->{$opt};
4366             $changes = 1;
4367         }
4368     }
4369
4370     if ($changes) {
4371         PVE::QemuConfig->write_config($vmid, $conf);
4372     }
4373
4374     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4375
4376     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4377     foreach my $opt (sort keys %$pending_delete_hash) {
4378         next if $selection && !$selection->{$opt};
4379         my $force = $pending_delete_hash->{$opt}->{force};
4380         eval {
4381             if ($opt eq 'hotplug') {
4382                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4383             } elsif ($opt eq 'tablet') {
4384                 die "skip\n" if !$hotplug_features->{usb};
4385                 if ($defaults->{tablet}) {
4386                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4387                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4388                         if $arch eq 'aarch64';
4389                 } else {
4390                     vm_deviceunplug($vmid, $conf, 'tablet');
4391                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4392                 }
4393             } elsif ($opt =~ m/^usb\d+/) {
4394                 die "skip\n";
4395                 # since we cannot reliably hot unplug usb devices we are disabling it
4396                 #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4397                 #vm_deviceunplug($vmid, $conf, $opt);
4398             } elsif ($opt eq 'vcpus') {
4399                 die "skip\n" if !$hotplug_features->{cpu};
4400                 qemu_cpu_hotplug($vmid, $conf, undef);
4401             } elsif ($opt eq 'balloon') {
4402                 # enable balloon device is not hotpluggable
4403                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4404                 # here we reset the ballooning value to memory
4405                 my $balloon = $conf->{memory} || $defaults->{memory};
4406                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4407             } elsif ($fast_plug_option->{$opt}) {
4408                 # do nothing
4409             } elsif ($opt =~ m/^net(\d+)$/) {
4410                 die "skip\n" if !$hotplug_features->{network};
4411                 vm_deviceunplug($vmid, $conf, $opt);
4412             } elsif (is_valid_drivename($opt)) {
4413                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4414                 vm_deviceunplug($vmid, $conf, $opt);
4415                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4416             } elsif ($opt =~ m/^memory$/) {
4417                 die "skip\n" if !$hotplug_features->{memory};
4418                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4419             } elsif ($opt eq 'cpuunits') {
4420                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4421             } elsif ($opt eq 'cpulimit') {
4422                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4423             } else {
4424                 die "skip\n";
4425             }
4426         };
4427         if (my $err = $@) {
4428             &$add_error($opt, $err) if $err ne "skip\n";
4429         } else {
4430             delete $conf->{$opt};
4431             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4432         }
4433     }
4434
4435     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4436     $apply_pending_cloudinit = sub {
4437         return if $apply_pending_cloudinit_done; # once is enough
4438         $apply_pending_cloudinit_done = 1; # once is enough
4439
4440         my ($key, $value) = @_;
4441
4442         my @cloudinit_opts = keys %$confdesc_cloudinit;
4443         foreach my $opt (keys %{$conf->{pending}}) {
4444             next if !grep { $_ eq $opt } @cloudinit_opts;
4445             $conf->{$opt} = delete $conf->{pending}->{$opt};
4446         }
4447
4448         my $new_conf = { %$conf };
4449         $new_conf->{$key} = $value;
4450         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4451     };
4452
4453     foreach my $opt (keys %{$conf->{pending}}) {
4454         next if $selection && !$selection->{$opt};
4455         my $value = $conf->{pending}->{$opt};
4456         eval {
4457             if ($opt eq 'hotplug') {
4458                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4459             } elsif ($opt eq 'tablet') {
4460                 die "skip\n" if !$hotplug_features->{usb};
4461                 if ($value == 1) {
4462                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4463                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4464                         if $arch eq 'aarch64';
4465                 } elsif ($value == 0) {
4466                     vm_deviceunplug($vmid, $conf, 'tablet');
4467                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4468                 }
4469             } elsif ($opt =~ m/^usb\d+$/) {
4470                 die "skip\n";
4471                 # since we cannot reliably hot unplug usb devices we disable it for now
4472                 #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4473                 #my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4474                 #die "skip\n" if !$d;
4475                 #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4476             } elsif ($opt eq 'vcpus') {
4477                 die "skip\n" if !$hotplug_features->{cpu};
4478                 qemu_cpu_hotplug($vmid, $conf, $value);
4479             } elsif ($opt eq 'balloon') {
4480                 # enable/disable balloning device is not hotpluggable
4481                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4482                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4483                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4484
4485                 # allow manual ballooning if shares is set to zero
4486                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4487                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4488                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4489                 }
4490             } elsif ($opt =~ m/^net(\d+)$/) {
4491                 # some changes can be done without hotplug
4492                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4493                                     $vmid, $opt, $value, $arch, $machine_type);
4494             } elsif (is_valid_drivename($opt)) {
4495                 die "skip\n" if $opt eq 'efidisk0';
4496                 # some changes can be done without hotplug
4497                 my $drive = parse_drive($opt, $value);
4498                 if (drive_is_cloudinit($drive)) {
4499                     &$apply_pending_cloudinit($opt, $value);
4500                 }
4501                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4502                                      $vmid, $opt, $value, $arch, $machine_type);
4503             } elsif ($opt =~ m/^memory$/) { #dimms
4504                 die "skip\n" if !$hotplug_features->{memory};
4505                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4506             } elsif ($opt eq 'cpuunits') {
4507                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4508             } elsif ($opt eq 'cpulimit') {
4509                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4510                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4511             } else {
4512                 die "skip\n";  # skip non-hot-pluggable options
4513             }
4514         };
4515         if (my $err = $@) {
4516             &$add_error($opt, $err) if $err ne "skip\n";
4517         } else {
4518             $conf->{$opt} = $value;
4519             delete $conf->{pending}->{$opt};
4520         }
4521     }
4522
4523     PVE::QemuConfig->write_config($vmid, $conf);
4524 }
4525
4526 sub try_deallocate_drive {
4527     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4528
4529     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4530         my $volid = $drive->{file};
4531         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4532             my $sid = PVE::Storage::parse_volume_id($volid);
4533             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4534
4535             # check if the disk is really unused
4536             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4537                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4538             PVE::Storage::vdisk_free($storecfg, $volid);
4539             return 1;
4540         } else {
4541             # If vm is not owner of this disk remove from config
4542             return 1;
4543         }
4544     }
4545
4546     return;
4547 }
4548
4549 sub vmconfig_delete_or_detach_drive {
4550     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4551
4552     my $drive = parse_drive($opt, $conf->{$opt});
4553
4554     my $rpcenv = PVE::RPCEnvironment::get();
4555     my $authuser = $rpcenv->get_user();
4556
4557     if ($force) {
4558         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4559         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4560     } else {
4561         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4562     }
4563 }
4564
4565
4566
4567 sub vmconfig_apply_pending {
4568     my ($vmid, $conf, $storecfg, $errors) = @_;
4569
4570     my $add_apply_error = sub {
4571         my ($opt, $msg) = @_;
4572         my $err_msg = "unable to apply pending change $opt : $msg";
4573         $errors->{$opt} = $err_msg;
4574         warn $err_msg;
4575     };
4576
4577     # cold plug
4578
4579     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4580     foreach my $opt (sort keys %$pending_delete_hash) {
4581         my $force = $pending_delete_hash->{$opt}->{force};
4582         eval {
4583             if ($opt =~ m/^unused/) {
4584                 die "internal error";
4585             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4586                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4587             }
4588         };
4589         if (my $err = $@) {
4590             $add_apply_error->($opt, $err);
4591         } else {
4592             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4593             delete $conf->{$opt};
4594         }
4595     }
4596
4597     PVE::QemuConfig->cleanup_pending($conf);
4598
4599     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4600         next if $opt eq 'delete'; # just to be sure
4601         eval {
4602             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4603                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4604             }
4605         };
4606         if (my $err = $@) {
4607             $add_apply_error->($opt, $err);
4608         } else {
4609             $conf->{$opt} = delete $conf->{pending}->{$opt};
4610         }
4611     }
4612
4613     # write all changes at once to avoid unnecessary i/o
4614     PVE::QemuConfig->write_config($vmid, $conf);
4615 }
4616
4617 sub vmconfig_update_net {
4618     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4619
4620     my $newnet = parse_net($value);
4621
4622     if ($conf->{$opt}) {
4623         my $oldnet = parse_net($conf->{$opt});
4624
4625         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4626             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4627             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4628             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4629
4630             # for non online change, we try to hot-unplug
4631             die "skip\n" if !$hotplug;
4632             vm_deviceunplug($vmid, $conf, $opt);
4633         } else {
4634
4635             die "internal error" if $opt !~ m/net(\d+)/;
4636             my $iface = "tap${vmid}i$1";
4637
4638             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4639                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4640                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4641                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4642                 PVE::Network::tap_unplug($iface);
4643
4644                 if ($have_sdn) {
4645                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4646                 } else {
4647                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4648                 }
4649             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4650                 # Rate can be applied on its own but any change above needs to
4651                 # include the rate in tap_plug since OVS resets everything.
4652                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4653             }
4654
4655             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4656                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4657             }
4658
4659             return 1;
4660         }
4661     }
4662
4663     if ($hotplug) {
4664         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4665     } else {
4666         die "skip\n";
4667     }
4668 }
4669
4670 sub vmconfig_update_disk {
4671     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4672
4673     my $drive = parse_drive($opt, $value);
4674
4675     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
4676         my $media = $drive->{media} || 'disk';
4677         my $oldmedia = $old_drive->{media} || 'disk';
4678         die "unable to change media type\n" if $media ne $oldmedia;
4679
4680         if (!drive_is_cdrom($old_drive)) {
4681
4682             if ($drive->{file} ne $old_drive->{file}) {
4683
4684                 die "skip\n" if !$hotplug;
4685
4686                 # unplug and register as unused
4687                 vm_deviceunplug($vmid, $conf, $opt);
4688                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4689
4690             } else {
4691                 # update existing disk
4692
4693                 # skip non hotpluggable value
4694                 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4695                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4696                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4697                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
4698                     safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
4699                     die "skip\n";
4700                 }
4701
4702                 # apply throttle
4703                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4704                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4705                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4706                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4707                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4708                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4709                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4710                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4711                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4712                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4713                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4714                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4715                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4716                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4717                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4718                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4719                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4720                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4721
4722                     qemu_block_set_io_throttle(
4723                         $vmid,"drive-$opt",
4724                         ($drive->{mbps} || 0)*1024*1024,
4725                         ($drive->{mbps_rd} || 0)*1024*1024,
4726                         ($drive->{mbps_wr} || 0)*1024*1024,
4727                          $drive->{iops} || 0,
4728                          $drive->{iops_rd} || 0,
4729                          $drive->{iops_wr} || 0,
4730                         ($drive->{mbps_max} || 0)*1024*1024,
4731                         ($drive->{mbps_rd_max} || 0)*1024*1024,
4732                         ($drive->{mbps_wr_max} || 0)*1024*1024,
4733                          $drive->{iops_max} || 0,
4734                          $drive->{iops_rd_max} || 0,
4735                          $drive->{iops_wr_max} || 0,
4736                          $drive->{bps_max_length} || 1,
4737                          $drive->{bps_rd_max_length} || 1,
4738                          $drive->{bps_wr_max_length} || 1,
4739                          $drive->{iops_max_length} || 1,
4740                          $drive->{iops_rd_max_length} || 1,
4741                          $drive->{iops_wr_max_length} || 1,
4742                     );
4743
4744                 }
4745
4746                 return 1;
4747             }
4748
4749         } else { # cdrom
4750
4751             if ($drive->{file} eq 'none') {
4752                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4753                 if (drive_is_cloudinit($old_drive)) {
4754                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4755                 }
4756             } else {
4757                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4758
4759                 # force eject if locked
4760                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4761
4762                 if ($path) {
4763                     mon_cmd($vmid, "blockdev-change-medium",
4764                         id => "$opt", filename => "$path");
4765                 }
4766             }
4767
4768             return 1;
4769         }
4770     }
4771
4772     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4773     # hotplug new disks
4774     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4775     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
4776 }
4777
4778 # called in locked context by incoming migration
4779 sub vm_migrate_get_nbd_disks {
4780     my ($storecfg, $conf, $replicated_volumes) = @_;
4781
4782     my $local_volumes = {};
4783     PVE::QemuConfig->foreach_volume($conf, sub {
4784         my ($ds, $drive) = @_;
4785
4786         return if drive_is_cdrom($drive);
4787
4788         my $volid = $drive->{file};
4789
4790         return if !$volid;
4791
4792         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4793
4794         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4795         return if $scfg->{shared};
4796
4797         # replicated disks re-use existing state via bitmap
4798         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
4799         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
4800     });
4801     return $local_volumes;
4802 }
4803
4804 # called in locked context by incoming migration
4805 sub vm_migrate_alloc_nbd_disks {
4806     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
4807
4808     my $format = undef;
4809
4810     my $nbd = {};
4811     foreach my $opt (sort keys %$source_volumes) {
4812         my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
4813
4814         if ($use_existing) {
4815             $nbd->{$opt}->{drivestr} = print_drive($drive);
4816             $nbd->{$opt}->{volid} = $volid;
4817             $nbd->{$opt}->{replicated} = 1;
4818             next;
4819         }
4820
4821         # If a remote storage is specified and the format of the original
4822         # volume is not available there, fall back to the default format.
4823         # Otherwise use the same format as the original.
4824         if (!$storagemap->{identity}) {
4825             $storeid = map_storage($storagemap, $storeid);
4826             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4827             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4828             my $fileFormat = qemu_img_format($scfg, $volname);
4829             $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
4830         } else {
4831             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4832             $format = qemu_img_format($scfg, $volname);
4833         }
4834
4835         my $size = $drive->{size} / 1024;
4836         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
4837         my $newdrive = $drive;
4838         $newdrive->{format} = $format;
4839         $newdrive->{file} = $newvolid;
4840         my $drivestr = print_drive($newdrive);
4841         $nbd->{$opt}->{drivestr} = $drivestr;
4842         $nbd->{$opt}->{volid} = $newvolid;
4843     }
4844
4845     return $nbd;
4846 }
4847
4848 # see vm_start_nolock for parameters, additionally:
4849 # migrate_opts:
4850 #   storagemap = parsed storage map for allocating NBD disks
4851 sub vm_start {
4852     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
4853
4854     return PVE::QemuConfig->lock_config($vmid, sub {
4855         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
4856
4857         die "you can't start a vm if it's a template\n"
4858             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
4859
4860         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
4861
4862         PVE::QemuConfig->check_lock($conf)
4863             if !($params->{skiplock} || $has_suspended_lock);
4864
4865         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
4866
4867         die "VM $vmid already running\n" if check_running($vmid, undef, $migrate_opts->{migratedfrom});
4868
4869         if (my $storagemap = $migrate_opts->{storagemap}) {
4870             my $replicated = $migrate_opts->{replicated_volumes};
4871             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
4872             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
4873
4874             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
4875                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
4876             }
4877         }
4878
4879         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
4880     });
4881 }
4882
4883
4884 # params:
4885 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
4886 #   skiplock => 0/1, skip checking for config lock
4887 #   skiptemplate => 0/1, skip checking whether VM is template
4888 #   forcemachine => to force Qemu machine (rollback/migration)
4889 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
4890 #   timeout => in seconds
4891 #   paused => start VM in paused state (backup)
4892 #   resume => resume from hibernation
4893 # migrate_opts:
4894 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
4895 #   migratedfrom => source node
4896 #   spice_ticket => used for spice migration, passed via tunnel/stdin
4897 #   network => CIDR of migration network
4898 #   type => secure/insecure - tunnel over encrypted connection or plain-text
4899 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
4900 #   replicated_volumes = which volids should be re-used with bitmaps for nbd migration
4901 sub vm_start_nolock {
4902     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
4903
4904     my $statefile = $params->{statefile};
4905     my $resume = $params->{resume};
4906
4907     my $migratedfrom = $migrate_opts->{migratedfrom};
4908     my $migration_type = $migrate_opts->{type};
4909
4910     my $res = {};
4911
4912     # clean up leftover reboot request files
4913     eval { clear_reboot_request($vmid); };
4914     warn $@ if $@;
4915
4916     if (!$statefile && scalar(keys %{$conf->{pending}})) {
4917         vmconfig_apply_pending($vmid, $conf, $storecfg);
4918         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4919     }
4920
4921     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4922
4923     my $defaults = load_defaults();
4924
4925     # set environment variable useful inside network script
4926     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4927
4928     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
4929
4930     my $forcemachine = $params->{forcemachine};
4931     my $forcecpu = $params->{forcecpu};
4932     if ($resume) {
4933         # enforce machine and CPU type on suspended vm to ensure HW compatibility
4934         $forcemachine = $conf->{runningmachine};
4935         $forcecpu = $conf->{runningcpu};
4936         print "Resuming suspended VM\n";
4937     }
4938
4939     my ($cmd, $vollist, $spice_port) =
4940         config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
4941
4942     my $migration_ip;
4943     my $get_migration_ip = sub {
4944         my ($nodename) = @_;
4945
4946         return $migration_ip if defined($migration_ip);
4947
4948         my $cidr = $migrate_opts->{network};
4949
4950         if (!defined($cidr)) {
4951             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4952             $cidr = $dc_conf->{migration}->{network};
4953         }
4954
4955         if (defined($cidr)) {
4956             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
4957
4958             die "could not get IP: no address configured on local " .
4959                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
4960
4961             die "could not get IP: multiple addresses configured on local " .
4962                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
4963
4964             $migration_ip = @$ips[0];
4965         }
4966
4967         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
4968             if !defined($migration_ip);
4969
4970         return $migration_ip;
4971     };
4972
4973     my $migrate_uri;
4974     if ($statefile) {
4975         if ($statefile eq 'tcp') {
4976             my $localip = "localhost";
4977             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4978             my $nodename = nodename();
4979
4980             if (!defined($migration_type)) {
4981                 if (defined($datacenterconf->{migration}->{type})) {
4982                     $migration_type = $datacenterconf->{migration}->{type};
4983                 } else {
4984                     $migration_type = 'secure';
4985                 }
4986             }
4987
4988             if ($migration_type eq 'insecure') {
4989                 $localip = $get_migration_ip->($nodename);
4990                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4991             }
4992
4993             my $pfamily = PVE::Tools::get_host_address_family($nodename);
4994             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4995             $migrate_uri = "tcp:${localip}:${migrate_port}";
4996             push @$cmd, '-incoming', $migrate_uri;
4997             push @$cmd, '-S';
4998
4999         } elsif ($statefile eq 'unix') {
5000             # should be default for secure migrations as a ssh TCP forward
5001             # tunnel is not deterministic reliable ready and fails regurarly
5002             # to set up in time, so use UNIX socket forwards
5003             my $socket_addr = "/run/qemu-server/$vmid.migrate";
5004             unlink $socket_addr;
5005
5006             $migrate_uri = "unix:$socket_addr";
5007
5008             push @$cmd, '-incoming', $migrate_uri;
5009             push @$cmd, '-S';
5010
5011         } elsif (-e $statefile) {
5012             push @$cmd, '-loadstate', $statefile;
5013         } else {
5014             my $statepath = PVE::Storage::path($storecfg, $statefile);
5015             push @$vollist, $statefile;
5016             push @$cmd, '-loadstate', $statepath;
5017         }
5018     } elsif ($params->{paused}) {
5019         push @$cmd, '-S';
5020     }
5021
5022     # host pci devices
5023     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5024       my $d = parse_hostpci($conf->{"hostpci$i"});
5025       next if !$d;
5026       my $pcidevices = $d->{pciid};
5027       foreach my $pcidevice (@$pcidevices) {
5028             my $pciid = $pcidevice->{id};
5029
5030             my $info = PVE::SysFSTools::pci_device_info("$pciid");
5031             die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5032             die "no pci device info for device '$pciid'\n" if !$info;
5033
5034             if ($d->{mdev}) {
5035                 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5036                 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5037             } else {
5038                 die "can't unbind/bind PCI group to VFIO '$pciid'\n"
5039                     if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5040                 die "can't reset PCI device '$pciid'\n"
5041                     if $info->{has_fl_reset} && !PVE::SysFSTools::pci_dev_reset($info);
5042             }
5043       }
5044     }
5045
5046     PVE::Storage::activate_volumes($storecfg, $vollist);
5047
5048     eval {
5049         run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5050             outfunc => sub {}, errfunc => sub {});
5051     };
5052     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5053     # timeout should be more than enough here...
5054     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5055
5056     my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5057                                               : $defaults->{cpuunits};
5058
5059     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5060     my %run_params = (
5061         timeout => $statefile ? undef : $start_timeout,
5062         umask => 0077,
5063         noerr => 1,
5064     );
5065
5066     # when migrating, prefix QEMU output so other side can pick up any
5067     # errors that might occur and show the user
5068     if ($migratedfrom) {
5069         $run_params{quiet} = 1;
5070         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5071     }
5072
5073     my %properties = (
5074         Slice => 'qemu.slice',
5075         KillMode => 'none',
5076         CPUShares => $cpuunits
5077     );
5078
5079     if (my $cpulimit = $conf->{cpulimit}) {
5080         $properties{CPUQuota} = int($cpulimit * 100);
5081     }
5082     $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5083
5084     my $run_qemu = sub {
5085         PVE::Tools::run_fork sub {
5086             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5087
5088             my $exitcode = run_command($cmd, %run_params);
5089             die "QEMU exited with code $exitcode\n" if $exitcode;
5090         };
5091     };
5092
5093     if ($conf->{hugepages}) {
5094
5095         my $code = sub {
5096             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5097             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5098
5099             PVE::QemuServer::Memory::hugepages_mount();
5100             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5101
5102             eval { $run_qemu->() };
5103             if (my $err = $@) {
5104                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5105                     if !$conf->{keephugepages};
5106                 die $err;
5107             }
5108
5109             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5110                 if !$conf->{keephugepages};
5111         };
5112         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5113
5114     } else {
5115         eval { $run_qemu->() };
5116     }
5117
5118     if (my $err = $@) {
5119         # deactivate volumes if start fails
5120         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5121         die "start failed: $err";
5122     }
5123
5124     print "migration listens on $migrate_uri\n" if $migrate_uri;
5125     $res->{migrate_uri} = $migrate_uri;
5126
5127     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5128         eval { mon_cmd($vmid, "cont"); };
5129         warn $@ if $@;
5130     }
5131
5132     #start nbd server for storage migration
5133     if (my $nbd = $migrate_opts->{nbd}) {
5134         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5135
5136         my $migrate_storage_uri;
5137         # nbd_protocol_version > 0 for unix socket support
5138         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5139             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5140             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5141             $migrate_storage_uri = "nbd:unix:$socket_path";
5142         } else {
5143             my $nodename = nodename();
5144             my $localip = $get_migration_ip->($nodename);
5145             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5146             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5147
5148             mon_cmd($vmid, "nbd-server-start", addr => {
5149                 type => 'inet',
5150                 data => {
5151                     host => "${localip}",
5152                     port => "${storage_migrate_port}",
5153                 },
5154             });
5155             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5156             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5157         }
5158
5159         $res->{migrate_storage_uri} = $migrate_storage_uri;
5160
5161         foreach my $opt (sort keys %$nbd) {
5162             my $drivestr = $nbd->{$opt}->{drivestr};
5163             my $volid = $nbd->{$opt}->{volid};
5164             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5165             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5166             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5167             print "re-using replicated volume: $opt - $volid\n"
5168                 if $nbd->{$opt}->{replicated};
5169
5170             $res->{drives}->{$opt} = $nbd->{$opt};
5171             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5172         }
5173     }
5174
5175     if ($migratedfrom) {
5176         eval {
5177             set_migration_caps($vmid);
5178         };
5179         warn $@ if $@;
5180
5181         if ($spice_port) {
5182             print "spice listens on port $spice_port\n";
5183             $res->{spice_port} = $spice_port;
5184             if ($migrate_opts->{spice_ticket}) {
5185                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5186                     $migrate_opts->{spice_ticket});
5187                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5188             }
5189         }
5190
5191     } else {
5192         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5193             if !$statefile && $conf->{balloon};
5194
5195         foreach my $opt (keys %$conf) {
5196             next if $opt !~  m/^net\d+$/;
5197             my $nicconf = parse_net($conf->{$opt});
5198             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5199         }
5200     }
5201
5202     mon_cmd($vmid, 'qom-set',
5203                 path => "machine/peripheral/balloon0",
5204                 property => "guest-stats-polling-interval",
5205                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5206
5207     if ($resume) {
5208         print "Resumed VM, removing state\n";
5209         if (my $vmstate = $conf->{vmstate}) {
5210             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5211             PVE::Storage::vdisk_free($storecfg, $vmstate);
5212         }
5213         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5214         PVE::QemuConfig->write_config($vmid, $conf);
5215     }
5216
5217     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5218
5219     return $res;
5220 }
5221
5222 sub vm_commandline {
5223     my ($storecfg, $vmid, $snapname) = @_;
5224
5225     my $conf = PVE::QemuConfig->load_config($vmid);
5226     my $forcemachine;
5227     my $forcecpu;
5228
5229     if ($snapname) {
5230         my $snapshot = $conf->{snapshots}->{$snapname};
5231         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5232
5233         # check for machine or CPU overrides in snapshot
5234         $forcemachine = $snapshot->{runningmachine};
5235         $forcecpu = $snapshot->{runningcpu};
5236
5237         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5238
5239         $conf = $snapshot;
5240     }
5241
5242     my $defaults = load_defaults();
5243
5244     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults,
5245         $forcemachine, $forcecpu);
5246
5247     return PVE::Tools::cmd2string($cmd);
5248 }
5249
5250 sub vm_reset {
5251     my ($vmid, $skiplock) = @_;
5252
5253     PVE::QemuConfig->lock_config($vmid, sub {
5254
5255         my $conf = PVE::QemuConfig->load_config($vmid);
5256
5257         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5258
5259         mon_cmd($vmid, "system_reset");
5260     });
5261 }
5262
5263 sub get_vm_volumes {
5264     my ($conf) = @_;
5265
5266     my $vollist = [];
5267     foreach_volid($conf, sub {
5268         my ($volid, $attr) = @_;
5269
5270         return if $volid =~ m|^/|;
5271
5272         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5273         return if !$sid;
5274
5275         push @$vollist, $volid;
5276     });
5277
5278     return $vollist;
5279 }
5280
5281 sub vm_stop_cleanup {
5282     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5283
5284     eval {
5285
5286         if (!$keepActive) {
5287             my $vollist = get_vm_volumes($conf);
5288             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5289         }
5290
5291         foreach my $ext (qw(mon qmp pid vnc qga)) {
5292             unlink "/var/run/qemu-server/${vmid}.$ext";
5293         }
5294
5295         if ($conf->{ivshmem}) {
5296             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5297             # just delete it for now, VMs which have this already open do not
5298             # are affected, but new VMs will get a separated one. If this
5299             # becomes an issue we either add some sort of ref-counting or just
5300             # add a "don't delete on stop" flag to the ivshmem format.
5301             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5302         }
5303
5304         foreach my $key (keys %$conf) {
5305             next if $key !~ m/^hostpci(\d+)$/;
5306             my $hostpciindex = $1;
5307             my $d = parse_hostpci($conf->{$key});
5308             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5309
5310             foreach my $pci (@{$d->{pciid}}) {
5311                 my $pciid = $pci->{id};
5312                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5313             }
5314         }
5315
5316         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5317     };
5318     warn $@ if $@; # avoid errors - just warn
5319 }
5320
5321 # call only in locked context
5322 sub _do_vm_stop {
5323     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5324
5325     my $pid = check_running($vmid, $nocheck);
5326     return if !$pid;
5327
5328     my $conf;
5329     if (!$nocheck) {
5330         $conf = PVE::QemuConfig->load_config($vmid);
5331         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5332         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5333             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5334             $timeout = $opts->{down} if $opts->{down};
5335         }
5336         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5337     }
5338
5339     eval {
5340         if ($shutdown) {
5341             if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5342                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5343             } else {
5344                 mon_cmd($vmid, "system_powerdown");
5345             }
5346         } else {
5347             mon_cmd($vmid, "quit");
5348         }
5349     };
5350     my $err = $@;
5351
5352     if (!$err) {
5353         $timeout = 60 if !defined($timeout);
5354
5355         my $count = 0;
5356         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5357             $count++;
5358             sleep 1;
5359         }
5360
5361         if ($count >= $timeout) {
5362             if ($force) {
5363                 warn "VM still running - terminating now with SIGTERM\n";
5364                 kill 15, $pid;
5365             } else {
5366                 die "VM quit/powerdown failed - got timeout\n";
5367             }
5368         } else {
5369             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5370             return;
5371         }
5372     } else {
5373        if (!check_running($vmid, $nocheck)) {
5374             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5375             return;
5376        }
5377        if ($force) {
5378             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5379             kill 15, $pid;
5380         } else {
5381             die "VM quit/powerdown failed\n";
5382         }
5383     }
5384
5385     # wait again
5386     $timeout = 10;
5387
5388     my $count = 0;
5389     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5390         $count++;
5391         sleep 1;
5392     }
5393
5394     if ($count >= $timeout) {
5395         warn "VM still running - terminating now with SIGKILL\n";
5396         kill 9, $pid;
5397         sleep 1;
5398     }
5399
5400     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5401 }
5402
5403 # Note: use $nocheck to skip tests if VM configuration file exists.
5404 # We need that when migration VMs to other nodes (files already moved)
5405 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5406 sub vm_stop {
5407     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5408
5409     $force = 1 if !defined($force) && !$shutdown;
5410
5411     if ($migratedfrom){
5412         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5413         kill 15, $pid if $pid;
5414         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5415         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5416         return;
5417     }
5418
5419     PVE::QemuConfig->lock_config($vmid, sub {
5420         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5421    });
5422 }
5423
5424 sub vm_reboot {
5425     my ($vmid, $timeout) = @_;
5426
5427     PVE::QemuConfig->lock_config($vmid, sub {
5428         eval {
5429
5430             # only reboot if running, as qmeventd starts it again on a stop event
5431             return if !check_running($vmid);
5432
5433             create_reboot_request($vmid);
5434
5435             my $storecfg = PVE::Storage::config();
5436             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5437
5438         };
5439         if (my $err = $@) {
5440             # avoid that the next normal shutdown will be confused for a reboot
5441             clear_reboot_request($vmid);
5442             die $err;
5443         }
5444    });
5445 }
5446
5447 # note: if using the statestorage parameter, the caller has to check privileges
5448 sub vm_suspend {
5449     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5450
5451     my $conf;
5452     my $path;
5453     my $storecfg;
5454     my $vmstate;
5455
5456     PVE::QemuConfig->lock_config($vmid, sub {
5457
5458         $conf = PVE::QemuConfig->load_config($vmid);
5459
5460         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5461         PVE::QemuConfig->check_lock($conf)
5462             if !($skiplock || $is_backing_up);
5463
5464         die "cannot suspend to disk during backup\n"
5465             if $is_backing_up && $includestate;
5466
5467         if ($includestate) {
5468             $conf->{lock} = 'suspending';
5469             my $date = strftime("%Y-%m-%d", localtime(time()));
5470             $storecfg = PVE::Storage::config();
5471             if (!$statestorage) {
5472                 $statestorage = find_vmstate_storage($conf, $storecfg);
5473                 # check permissions for the storage
5474                 my $rpcenv = PVE::RPCEnvironment::get();
5475                 if ($rpcenv->{type} ne 'cli') {
5476                     my $authuser = $rpcenv->get_user();
5477                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5478                 }
5479             }
5480
5481
5482             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
5483                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5484             $path = PVE::Storage::path($storecfg, $vmstate);
5485             PVE::QemuConfig->write_config($vmid, $conf);
5486         } else {
5487             mon_cmd($vmid, "stop");
5488         }
5489     });
5490
5491     if ($includestate) {
5492         # save vm state
5493         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5494
5495         eval {
5496             mon_cmd($vmid, "savevm-start", statefile => $path);
5497             for(;;) {
5498                 my $state = mon_cmd($vmid, "query-savevm");
5499                 if (!$state->{status}) {
5500                     die "savevm not active\n";
5501                 } elsif ($state->{status} eq 'active') {
5502                     sleep(1);
5503                     next;
5504                 } elsif ($state->{status} eq 'completed') {
5505                     print "State saved, quitting\n";
5506                     last;
5507                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5508                     die "query-savevm failed with error '$state->{error}'\n"
5509                 } else {
5510                     die "query-savevm returned status '$state->{status}'\n";
5511                 }
5512             }
5513         };
5514         my $err = $@;
5515
5516         PVE::QemuConfig->lock_config($vmid, sub {
5517             $conf = PVE::QemuConfig->load_config($vmid);
5518             if ($err) {
5519                 # cleanup, but leave suspending lock, to indicate something went wrong
5520                 eval {
5521                     mon_cmd($vmid, "savevm-end");
5522                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5523                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5524                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
5525                     PVE::QemuConfig->write_config($vmid, $conf);
5526                 };
5527                 warn $@ if $@;
5528                 die $err;
5529             }
5530
5531             die "lock changed unexpectedly\n"
5532                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5533
5534             mon_cmd($vmid, "quit");
5535             $conf->{lock} = 'suspended';
5536             PVE::QemuConfig->write_config($vmid, $conf);
5537         });
5538     }
5539 }
5540
5541 sub vm_resume {
5542     my ($vmid, $skiplock, $nocheck) = @_;
5543
5544     PVE::QemuConfig->lock_config($vmid, sub {
5545         my $res = mon_cmd($vmid, 'query-status');
5546         my $resume_cmd = 'cont';
5547
5548         if ($res->{status} && $res->{status} eq 'suspended') {
5549             $resume_cmd = 'system_wakeup';
5550         }
5551
5552         if (!$nocheck) {
5553
5554             my $conf = PVE::QemuConfig->load_config($vmid);
5555
5556             PVE::QemuConfig->check_lock($conf)
5557                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5558         }
5559
5560         mon_cmd($vmid, $resume_cmd);
5561     });
5562 }
5563
5564 sub vm_sendkey {
5565     my ($vmid, $skiplock, $key) = @_;
5566
5567     PVE::QemuConfig->lock_config($vmid, sub {
5568
5569         my $conf = PVE::QemuConfig->load_config($vmid);
5570
5571         # there is no qmp command, so we use the human monitor command
5572         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5573         die $res if $res ne '';
5574     });
5575 }
5576
5577 # vzdump restore implementaion
5578
5579 sub tar_archive_read_firstfile {
5580     my $archive = shift;
5581
5582     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5583
5584     # try to detect archive type first
5585     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5586         die "unable to open file '$archive'\n";
5587     my $firstfile = <$fh>;
5588     kill 15, $pid;
5589     close $fh;
5590
5591     die "ERROR: archive contaions no data\n" if !$firstfile;
5592     chomp $firstfile;
5593
5594     return $firstfile;
5595 }
5596
5597 sub tar_restore_cleanup {
5598     my ($storecfg, $statfile) = @_;
5599
5600     print STDERR "starting cleanup\n";
5601
5602     if (my $fd = IO::File->new($statfile, "r")) {
5603         while (defined(my $line = <$fd>)) {
5604             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5605                 my $volid = $2;
5606                 eval {
5607                     if ($volid =~ m|^/|) {
5608                         unlink $volid || die 'unlink failed\n';
5609                     } else {
5610                         PVE::Storage::vdisk_free($storecfg, $volid);
5611                     }
5612                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5613                 };
5614                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5615             } else {
5616                 print STDERR "unable to parse line in statfile - $line";
5617             }
5618         }
5619         $fd->close();
5620     }
5621 }
5622
5623 sub restore_file_archive {
5624     my ($archive, $vmid, $user, $opts) = @_;
5625
5626     return restore_vma_archive($archive, $vmid, $user, $opts)
5627         if $archive eq '-';
5628
5629     my $info = PVE::Storage::archive_info($archive);
5630     my $format = $opts->{format} // $info->{format};
5631     my $comp = $info->{compression};
5632
5633     # try to detect archive format
5634     if ($format eq 'tar') {
5635         return restore_tar_archive($archive, $vmid, $user, $opts);
5636     } else {
5637         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5638     }
5639 }
5640
5641 # hepler to remove disks that will not be used after restore
5642 my $restore_cleanup_oldconf = sub {
5643     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
5644
5645     PVE::QemuConfig->foreach_volume($oldconf, sub {
5646         my ($ds, $drive) = @_;
5647
5648         return if drive_is_cdrom($drive, 1);
5649
5650         my $volid = $drive->{file};
5651         return if !$volid || $volid =~ m|^/|;
5652
5653         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
5654         return if !$path || !$owner || ($owner != $vmid);
5655
5656         # Note: only delete disk we want to restore
5657         # other volumes will become unused
5658         if ($virtdev_hash->{$ds}) {
5659             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
5660             if (my $err = $@) {
5661                 warn $err;
5662             }
5663         }
5664     });
5665
5666     # delete vmstate files, after the restore we have no snapshots anymore
5667     foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5668         my $snap = $oldconf->{snapshots}->{$snapname};
5669         if ($snap->{vmstate}) {
5670             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
5671             if (my $err = $@) {
5672                 warn $err;
5673             }
5674         }
5675     }
5676 };
5677
5678 # Helper to parse vzdump backup device hints
5679 #
5680 # $rpcenv: Environment, used to ckeck storage permissions
5681 # $user: User ID, to check storage permissions
5682 # $storecfg: Storage configuration
5683 # $fh: the file handle for reading the configuration
5684 # $devinfo: should contain device sizes for all backu-up'ed devices
5685 # $options: backup options (pool, default storage)
5686 #
5687 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
5688 my $parse_backup_hints = sub {
5689     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
5690
5691     my $virtdev_hash = {};
5692
5693     while (defined(my $line = <$fh>)) {
5694         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5695             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5696             die "archive does not contain data for drive '$virtdev'\n"
5697                 if !$devinfo->{$devname};
5698
5699             if (defined($options->{storage})) {
5700                 $storeid = $options->{storage} || 'local';
5701             } elsif (!$storeid) {
5702                 $storeid = 'local';
5703             }
5704             $format = 'raw' if !$format;
5705             $devinfo->{$devname}->{devname} = $devname;
5706             $devinfo->{$devname}->{virtdev} = $virtdev;
5707             $devinfo->{$devname}->{format} = $format;
5708             $devinfo->{$devname}->{storeid} = $storeid;
5709
5710             # check permission on storage
5711             my $pool = $options->{pool}; # todo: do we need that?
5712             if ($user ne 'root@pam') {
5713                 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5714             }
5715
5716             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5717         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
5718             my $virtdev = $1;
5719             my $drive = parse_drive($virtdev, $2);
5720             if (drive_is_cloudinit($drive)) {
5721                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5722                 $storeid = $options->{storage} if defined ($options->{storage});
5723                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5724                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
5725
5726                 $virtdev_hash->{$virtdev} = {
5727                     format => $format,
5728                     storeid => $storeid,
5729                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
5730                     is_cloudinit => 1,
5731                 };
5732             }
5733         }
5734     }
5735
5736     return $virtdev_hash;
5737 };
5738
5739 # Helper to allocate and activate all volumes required for a restore
5740 #
5741 # $storecfg: Storage configuration
5742 # $virtdev_hash: as returned by parse_backup_hints()
5743 #
5744 # Returns: { $virtdev => $volid }
5745 my $restore_allocate_devices = sub {
5746     my ($storecfg, $virtdev_hash, $vmid) = @_;
5747
5748     my $map = {};
5749     foreach my $virtdev (sort keys %$virtdev_hash) {
5750         my $d = $virtdev_hash->{$virtdev};
5751         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5752         my $storeid = $d->{storeid};
5753         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5754
5755         # test if requested format is supported
5756         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5757         my $supported = grep { $_ eq $d->{format} } @$validFormats;
5758         $d->{format} = $defFormat if !$supported;
5759
5760         my $name;
5761         if ($d->{is_cloudinit}) {
5762             $name = "vm-$vmid-cloudinit";
5763             $name .= ".$d->{format}" if $d->{format} ne 'raw';
5764         }
5765
5766         my $volid = PVE::Storage::vdisk_alloc(
5767             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
5768
5769         print STDERR "new volume ID is '$volid'\n";
5770         $d->{volid} = $volid;
5771
5772         PVE::Storage::activate_volumes($storecfg, [$volid]);
5773
5774         $map->{$virtdev} = $volid;
5775     }
5776
5777     return $map;
5778 };
5779
5780 my $restore_update_config_line = sub {
5781     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5782
5783     return if $line =~ m/^\#qmdump\#/;
5784     return if $line =~ m/^\#vzdump\#/;
5785     return if $line =~ m/^lock:/;
5786     return if $line =~ m/^unused\d+:/;
5787     return if $line =~ m/^parent:/;
5788
5789     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5790     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5791         # try to convert old 1.X settings
5792         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5793         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5794             my ($model, $macaddr) = split(/\=/, $devconfig);
5795             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5796             my $net = {
5797                 model => $model,
5798                 bridge => "vmbr$ind",
5799                 macaddr => $macaddr,
5800             };
5801             my $netstr = print_net($net);
5802
5803             print $outfd "net$cookie->{netcount}: $netstr\n";
5804             $cookie->{netcount}++;
5805         }
5806     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5807         my ($id, $netstr) = ($1, $2);
5808         my $net = parse_net($netstr);
5809         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5810         $netstr = print_net($net);
5811         print $outfd "$id: $netstr\n";
5812     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5813         my $virtdev = $1;
5814         my $value = $3;
5815         my $di = parse_drive($virtdev, $value);
5816         if (defined($di->{backup}) && !$di->{backup}) {
5817             print $outfd "#$line";
5818         } elsif ($map->{$virtdev}) {
5819             delete $di->{format}; # format can change on restore
5820             $di->{file} = $map->{$virtdev};
5821             $value = print_drive($di);
5822             print $outfd "$virtdev: $value\n";
5823         } else {
5824             print $outfd $line;
5825         }
5826     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5827         my $vmgenid = $1;
5828         if ($vmgenid ne '0') {
5829             # always generate a new vmgenid if there was a valid one setup
5830             $vmgenid = generate_uuid();
5831         }
5832         print $outfd "vmgenid: $vmgenid\n";
5833     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5834         my ($uuid, $uuid_str);
5835         UUID::generate($uuid);
5836         UUID::unparse($uuid, $uuid_str);
5837         my $smbios1 = parse_smbios1($2);
5838         $smbios1->{uuid} = $uuid_str;
5839         print $outfd $1.print_smbios1($smbios1)."\n";
5840     } else {
5841         print $outfd $line;
5842     }
5843 };
5844
5845 my $restore_deactivate_volumes = sub {
5846     my ($storecfg, $devinfo) = @_;
5847
5848     my $vollist = [];
5849     foreach my $devname (keys %$devinfo) {
5850         my $volid = $devinfo->{$devname}->{volid};
5851         push @$vollist, $volid if $volid;
5852     }
5853
5854     PVE::Storage::deactivate_volumes($storecfg, $vollist);
5855 };
5856
5857 my $restore_destroy_volumes = sub {
5858     my ($storecfg, $devinfo) = @_;
5859
5860     foreach my $devname (keys %$devinfo) {
5861         my $volid = $devinfo->{$devname}->{volid};
5862         next if !$volid;
5863         eval {
5864             if ($volid =~ m|^/|) {
5865                 unlink $volid || die 'unlink failed\n';
5866             } else {
5867                 PVE::Storage::vdisk_free($storecfg, $volid);
5868             }
5869             print STDERR "temporary volume '$volid' sucessfuly removed\n";
5870         };
5871         print STDERR "unable to cleanup '$volid' - $@" if $@;
5872     }
5873 };
5874
5875 sub scan_volids {
5876     my ($cfg, $vmid) = @_;
5877
5878     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5879
5880     my $volid_hash = {};
5881     foreach my $storeid (keys %$info) {
5882         foreach my $item (@{$info->{$storeid}}) {
5883             next if !($item->{volid} && $item->{size});
5884             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5885             $volid_hash->{$item->{volid}} = $item;
5886         }
5887     }
5888
5889     return $volid_hash;
5890 }
5891
5892 sub update_disk_config {
5893     my ($vmid, $conf, $volid_hash) = @_;
5894
5895     my $changes;
5896     my $prefix = "VM $vmid";
5897
5898     # used and unused disks
5899     my $referenced = {};
5900
5901     # Note: it is allowed to define multiple storages with same path (alias), so
5902     # we need to check both 'volid' and real 'path' (two different volid can point
5903     # to the same path).
5904
5905     my $referencedpath = {};
5906
5907     # update size info
5908     PVE::QemuConfig->foreach_volume($conf, sub {
5909         my ($opt, $drive) = @_;
5910
5911         my $volid = $drive->{file};
5912         return if !$volid;
5913         my $volume = $volid_hash->{$volid};
5914
5915         # mark volid as "in-use" for next step
5916         $referenced->{$volid} = 1;
5917         if ($volume && (my $path = $volume->{path})) {
5918             $referencedpath->{$path} = 1;
5919         }
5920
5921         return if drive_is_cdrom($drive);
5922         return if !$volume;
5923
5924         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
5925         if (defined($updated)) {
5926             $changes = 1;
5927             $conf->{$opt} = print_drive($updated);
5928             print "$prefix ($opt): $msg\n";
5929         }
5930     });
5931
5932     # remove 'unusedX' entry if volume is used
5933     PVE::QemuConfig->foreach_unused_volume($conf, sub {
5934         my ($opt, $drive) = @_;
5935
5936         my $volid = $drive->{file};
5937         return if !$volid;
5938
5939         my $path;
5940         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5941         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5942             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
5943             $changes = 1;
5944             delete $conf->{$opt};
5945         }
5946
5947         $referenced->{$volid} = 1;
5948         $referencedpath->{$path} = 1 if $path;
5949     });
5950
5951     foreach my $volid (sort keys %$volid_hash) {
5952         next if $volid =~ m/vm-$vmid-state-/;
5953         next if $referenced->{$volid};
5954         my $path = $volid_hash->{$volid}->{path};
5955         next if !$path; # just to be sure
5956         next if $referencedpath->{$path};
5957         $changes = 1;
5958         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
5959         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
5960         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5961     }
5962
5963     return $changes;
5964 }
5965
5966 sub rescan {
5967     my ($vmid, $nolock, $dryrun) = @_;
5968
5969     my $cfg = PVE::Storage::config();
5970
5971     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
5972     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
5973     foreach my $stor (keys %{$cfg->{ids}}) {
5974         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
5975     }
5976
5977     print "rescan volumes...\n";
5978     my $volid_hash = scan_volids($cfg, $vmid);
5979
5980     my $updatefn =  sub {
5981         my ($vmid) = @_;
5982
5983         my $conf = PVE::QemuConfig->load_config($vmid);
5984
5985         PVE::QemuConfig->check_lock($conf);
5986
5987         my $vm_volids = {};
5988         foreach my $volid (keys %$volid_hash) {
5989             my $info = $volid_hash->{$volid};
5990             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5991         }
5992
5993         my $changes = update_disk_config($vmid, $conf, $vm_volids);
5994
5995         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
5996     };
5997
5998     if (defined($vmid)) {
5999         if ($nolock) {
6000             &$updatefn($vmid);
6001         } else {
6002             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6003         }
6004     } else {
6005         my $vmlist = config_list();
6006         foreach my $vmid (keys %$vmlist) {
6007             if ($nolock) {
6008                 &$updatefn($vmid);
6009             } else {
6010                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6011             }
6012         }
6013     }
6014 }
6015
6016 sub restore_proxmox_backup_archive {
6017     my ($archive, $vmid, $user, $options) = @_;
6018
6019     my $storecfg = PVE::Storage::config();
6020
6021     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6022     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6023
6024     my $server = $scfg->{server};
6025     my $datastore = $scfg->{datastore};
6026     my $username = $scfg->{username} // 'root@pam';
6027     my $fingerprint = $scfg->{fingerprint};
6028     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6029
6030     my $repo = "$username\@$server:$datastore";
6031
6032     # This is only used for `pbs-restore`!
6033     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6034     local $ENV{PBS_PASSWORD} = $password;
6035     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6036
6037     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6038         PVE::Storage::parse_volname($storecfg, $archive);
6039
6040     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6041
6042     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6043
6044     my $tmpdir = "/var/tmp/vzdumptmp$$";
6045     rmtree $tmpdir;
6046     mkpath $tmpdir;
6047
6048     my $conffile = PVE::QemuConfig->config_file($vmid);
6049     my $tmpfn = "$conffile.$$.tmp";
6050      # disable interrupts (always do cleanups)
6051     local $SIG{INT} =
6052         local $SIG{TERM} =
6053         local $SIG{QUIT} =
6054         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6055
6056     # Note: $oldconf is undef if VM does not exists
6057     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6058     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6059
6060     my $rpcenv = PVE::RPCEnvironment::get();
6061     my $devinfo = {};
6062
6063     eval {
6064         # enable interrupts
6065         local $SIG{INT} =
6066             local $SIG{TERM} =
6067             local $SIG{QUIT} =
6068             local $SIG{HUP} =
6069             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6070
6071         my $cfgfn = "$tmpdir/qemu-server.conf";
6072         my $firewall_config_fn = "$tmpdir/fw.conf";
6073         my $index_fn = "$tmpdir/index.json";
6074
6075         my $cmd = "restore";
6076
6077         my $param = [$pbs_backup_name, "index.json", $index_fn];
6078         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6079         my $index = PVE::Tools::file_get_contents($index_fn);
6080         $index = decode_json($index);
6081
6082         # print Dumper($index);
6083         foreach my $info (@{$index->{files}}) {
6084             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6085                 my $devname = $1;
6086                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6087                     $devinfo->{$devname}->{size} = $1;
6088                 } else {
6089                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6090                 }
6091             }
6092         }
6093
6094         my $is_qemu_server_backup = scalar(
6095             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6096         );
6097         if (!$is_qemu_server_backup) {
6098             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6099         }
6100         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6101
6102         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6103         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6104
6105         if ($has_firewall_config) {
6106             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6107             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6108
6109             my $pve_firewall_dir = '/etc/pve/firewall';
6110             mkdir $pve_firewall_dir; # make sure the dir exists
6111             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6112         }
6113
6114         my $fh = IO::File->new($cfgfn, "r") ||
6115             die "unable to read qemu-server.conf - $!\n";
6116
6117         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6118
6119         # fixme: rate limit?
6120
6121         # create empty/temp config
6122         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6123
6124         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6125
6126         # allocate volumes
6127         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6128
6129         foreach my $virtdev (sort keys %$virtdev_hash) {
6130             my $d = $virtdev_hash->{$virtdev};
6131             next if $d->{is_cloudinit}; # no need to restore cloudinit
6132
6133             my $volid = $d->{volid};
6134
6135             my $path = PVE::Storage::path($storecfg, $volid);
6136
6137             # This is the ONLY user of the PBS_ env vars set on top of this function!
6138             my $pbs_restore_cmd = [
6139                 '/usr/bin/pbs-restore',
6140                 '--repository', $repo,
6141                 $pbs_backup_name,
6142                 "$d->{devname}.img.fidx",
6143                 $path,
6144                 '--verbose',
6145                 ];
6146
6147             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6148             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
6149
6150             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6151                 push @$pbs_restore_cmd, '--skip-zero';
6152             }
6153
6154             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6155             print "restore proxmox backup image: $dbg_cmdstring\n";
6156             run_command($pbs_restore_cmd);
6157         }
6158
6159         $fh->seek(0, 0) || die "seek failed - $!\n";
6160
6161         my $outfd = IO::File->new($tmpfn, "w") || die "unable to write config for VM $vmid\n";
6162
6163         my $cookie = { netcount => 0 };
6164         while (defined(my $line = <$fh>)) {
6165             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $options->{unique});
6166         }
6167
6168         $fh->close();
6169         $outfd->close();
6170     };
6171     my $err = $@;
6172
6173     $restore_deactivate_volumes->($storecfg, $devinfo);
6174
6175     rmtree $tmpdir;
6176
6177     if ($err) {
6178         unlink $tmpfn;
6179         $restore_destroy_volumes->($storecfg, $devinfo);
6180         die $err;
6181     }
6182
6183     rename($tmpfn, $conffile) ||
6184         die "unable to commit configuration file '$conffile'\n";
6185
6186     PVE::Cluster::cfs_update(); # make sure we read new file
6187
6188     eval { rescan($vmid, 1); };
6189     warn $@ if $@;
6190 }
6191
6192 sub restore_vma_archive {
6193     my ($archive, $vmid, $user, $opts, $comp) = @_;
6194
6195     my $readfrom = $archive;
6196
6197     my $cfg = PVE::Storage::config();
6198     my $commands = [];
6199     my $bwlimit = $opts->{bwlimit};
6200
6201     my $dbg_cmdstring = '';
6202     my $add_pipe = sub {
6203         my ($cmd) = @_;
6204         push @$commands, $cmd;
6205         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6206         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6207         $readfrom = '-';
6208     };
6209
6210     my $input = undef;
6211     if ($archive eq '-') {
6212         $input = '<&STDIN';
6213     } else {
6214         # If we use a backup from a PVE defined storage we also consider that
6215         # storage's rate limit:
6216         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6217         if (defined($volid)) {
6218             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6219             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6220             if ($readlimit) {
6221                 print STDERR "applying read rate limit: $readlimit\n";
6222                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6223                 $add_pipe->($cstream);
6224             }
6225         }
6226     }
6227
6228     if ($comp) {
6229         my $info = PVE::Storage::decompressor_info('vma', $comp);
6230         my $cmd = $info->{decompressor};
6231         push @$cmd, $readfrom;
6232         $add_pipe->($cmd);
6233     }
6234
6235     my $tmpdir = "/var/tmp/vzdumptmp$$";
6236     rmtree $tmpdir;
6237
6238     # disable interrupts (always do cleanups)
6239     local $SIG{INT} =
6240         local $SIG{TERM} =
6241         local $SIG{QUIT} =
6242         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6243
6244     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6245     POSIX::mkfifo($mapfifo, 0600);
6246     my $fifofh;
6247     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
6248
6249     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6250
6251     my $oldtimeout;
6252     my $timeout = 5;
6253
6254     my $devinfo = {};
6255
6256     my $rpcenv = PVE::RPCEnvironment::get();
6257
6258     my $conffile = PVE::QemuConfig->config_file($vmid);
6259     my $tmpfn = "$conffile.$$.tmp";
6260
6261     # Note: $oldconf is undef if VM does not exist
6262     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6263     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6264
6265     my %storage_limits;
6266
6267     my $print_devmap = sub {
6268         my $cfgfn = "$tmpdir/qemu-server.conf";
6269
6270         # we can read the config - that is already extracted
6271         my $fh = IO::File->new($cfgfn, "r") ||
6272             die "unable to read qemu-server.conf - $!\n";
6273
6274         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6275         if (-f $fwcfgfn) {
6276             my $pve_firewall_dir = '/etc/pve/firewall';
6277             mkdir $pve_firewall_dir; # make sure the dir exists
6278             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6279         }
6280
6281         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6282
6283         foreach my $key (keys %storage_limits) {
6284             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6285             next if !$limit;
6286             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6287             $storage_limits{$key} = $limit * 1024;
6288         }
6289
6290         foreach my $devname (keys %$devinfo) {
6291             die "found no device mapping information for device '$devname'\n"
6292                 if !$devinfo->{$devname}->{virtdev};
6293         }
6294
6295         # create empty/temp config
6296         if ($oldconf) {
6297             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6298             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6299         }
6300
6301         # allocate volumes
6302         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6303
6304         # print restore information to $fifofh
6305         foreach my $virtdev (sort keys %$virtdev_hash) {
6306             my $d = $virtdev_hash->{$virtdev};
6307             next if $d->{is_cloudinit}; # no need to restore cloudinit
6308
6309             my $storeid = $d->{storeid};
6310             my $volid = $d->{volid};
6311
6312             my $map_opts = '';
6313             if (my $limit = $storage_limits{$storeid}) {
6314                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6315             }
6316
6317             my $write_zeros = 1;
6318             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6319                 $write_zeros = 0;
6320             }
6321
6322             my $path = PVE::Storage::path($cfg, $volid);
6323
6324             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6325
6326             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6327         }
6328
6329         $fh->seek(0, 0) || die "seek failed - $!\n";
6330
6331         my $outfd = IO::File->new($tmpfn, "w") || die "unable to write config for VM $vmid\n";
6332
6333         my $cookie = { netcount => 0 };
6334         while (defined(my $line = <$fh>)) {
6335             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6336         }
6337
6338         $fh->close();
6339         $outfd->close();
6340     };
6341
6342     eval {
6343         # enable interrupts
6344         local $SIG{INT} =
6345             local $SIG{TERM} =
6346             local $SIG{QUIT} =
6347             local $SIG{HUP} =
6348             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6349         local $SIG{ALRM} = sub { die "got timeout\n"; };
6350
6351         $oldtimeout = alarm($timeout);
6352
6353         my $parser = sub {
6354             my $line = shift;
6355
6356             print "$line\n";
6357
6358             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6359                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6360                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6361             } elsif ($line =~ m/^CTIME: /) {
6362                 # we correctly received the vma config, so we can disable
6363                 # the timeout now for disk allocation (set to 10 minutes, so
6364                 # that we always timeout if something goes wrong)
6365                 alarm(600);
6366                 &$print_devmap();
6367                 print $fifofh "done\n";
6368                 my $tmp = $oldtimeout || 0;
6369                 $oldtimeout = undef;
6370                 alarm($tmp);
6371                 close($fifofh);
6372                 $fifofh = undef;
6373             }
6374         };
6375
6376         print "restore vma archive: $dbg_cmdstring\n";
6377         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6378     };
6379     my $err = $@;
6380
6381     alarm($oldtimeout) if $oldtimeout;
6382
6383     $restore_deactivate_volumes->($cfg, $devinfo);
6384
6385     close($fifofh) if $fifofh;
6386     unlink $mapfifo;
6387     rmtree $tmpdir;
6388
6389     if ($err) {
6390         unlink $tmpfn;
6391         $restore_destroy_volumes->($cfg, $devinfo);
6392         die $err;
6393     }
6394
6395     rename($tmpfn, $conffile) ||
6396         die "unable to commit configuration file '$conffile'\n";
6397
6398     PVE::Cluster::cfs_update(); # make sure we read new file
6399
6400     eval { rescan($vmid, 1); };
6401     warn $@ if $@;
6402 }
6403
6404 sub restore_tar_archive {
6405     my ($archive, $vmid, $user, $opts) = @_;
6406
6407     if ($archive ne '-') {
6408         my $firstfile = tar_archive_read_firstfile($archive);
6409         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
6410             if $firstfile ne 'qemu-server.conf';
6411     }
6412
6413     my $storecfg = PVE::Storage::config();
6414
6415     # avoid zombie disks when restoring over an existing VM -> cleanup first
6416     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6417     # skiplock=1 because qmrestore has set the 'create' lock itself already
6418     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6419     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6420
6421     my $tocmd = "/usr/lib/qemu-server/qmextract";
6422
6423     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6424     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6425     $tocmd .= ' --prealloc' if $opts->{prealloc};
6426     $tocmd .= ' --info' if $opts->{info};
6427
6428     # tar option "xf" does not autodetect compression when read from STDIN,
6429     # so we pipe to zcat
6430     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6431         PVE::Tools::shellquote("--to-command=$tocmd");
6432
6433     my $tmpdir = "/var/tmp/vzdumptmp$$";
6434     mkpath $tmpdir;
6435
6436     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6437     local $ENV{VZDUMP_VMID} = $vmid;
6438     local $ENV{VZDUMP_USER} = $user;
6439
6440     my $conffile = PVE::QemuConfig->config_file($vmid);
6441     my $tmpfn = "$conffile.$$.tmp";
6442
6443     # disable interrupts (always do cleanups)
6444     local $SIG{INT} =
6445         local $SIG{TERM} =
6446         local $SIG{QUIT} =
6447         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6448
6449     eval {
6450         # enable interrupts
6451         local $SIG{INT} =
6452             local $SIG{TERM} =
6453             local $SIG{QUIT} =
6454             local $SIG{HUP} =
6455             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6456
6457         if ($archive eq '-') {
6458             print "extracting archive from STDIN\n";
6459             run_command($cmd, input => "<&STDIN");
6460         } else {
6461             print "extracting archive '$archive'\n";
6462             run_command($cmd);
6463         }
6464
6465         return if $opts->{info};
6466
6467         # read new mapping
6468         my $map = {};
6469         my $statfile = "$tmpdir/qmrestore.stat";
6470         if (my $fd = IO::File->new($statfile, "r")) {
6471             while (defined (my $line = <$fd>)) {
6472                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6473                     $map->{$1} = $2 if $1;
6474                 } else {
6475                     print STDERR "unable to parse line in statfile - $line\n";
6476                 }
6477             }
6478             $fd->close();
6479         }
6480
6481         my $confsrc = "$tmpdir/qemu-server.conf";
6482
6483         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
6484
6485         my $outfd = IO::File->new($tmpfn, "w") || die "unable to write config for VM $vmid\n";
6486
6487         my $cookie = { netcount => 0 };
6488         while (defined (my $line = <$srcfd>)) {
6489             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6490         }
6491
6492         $srcfd->close();
6493         $outfd->close();
6494     };
6495     if (my $err = $@) {
6496         unlink $tmpfn;
6497         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6498         die $err;
6499     }
6500
6501     rmtree $tmpdir;
6502
6503     rename $tmpfn, $conffile ||
6504         die "unable to commit configuration file '$conffile'\n";
6505
6506     PVE::Cluster::cfs_update(); # make sure we read new file
6507
6508     eval { rescan($vmid, 1); };
6509     warn $@ if $@;
6510 };
6511
6512 sub foreach_storage_used_by_vm {
6513     my ($conf, $func) = @_;
6514
6515     my $sidhash = {};
6516
6517     PVE::QemuConfig->foreach_volume($conf, sub {
6518         my ($ds, $drive) = @_;
6519         return if drive_is_cdrom($drive);
6520
6521         my $volid = $drive->{file};
6522
6523         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6524         $sidhash->{$sid} = $sid if $sid;
6525     });
6526
6527     foreach my $sid (sort keys %$sidhash) {
6528         &$func($sid);
6529     }
6530 }
6531
6532 my $qemu_snap_storage = {
6533     rbd => 1,
6534 };
6535 sub do_snapshots_with_qemu {
6536     my ($storecfg, $volid) = @_;
6537
6538     my $storage_name = PVE::Storage::parse_volume_id($volid);
6539     my $scfg = $storecfg->{ids}->{$storage_name};
6540     die "could not find storage '$storage_name'\n" if !defined($scfg);
6541
6542     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6543         return 1;
6544     }
6545
6546     if ($volid =~ m/\.(qcow2|qed)$/){
6547         return 1;
6548     }
6549
6550     return;
6551 }
6552
6553 sub qga_check_running {
6554     my ($vmid, $nowarn) = @_;
6555
6556     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6557     if ($@) {
6558         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6559         return 0;
6560     }
6561     return 1;
6562 }
6563
6564 sub template_create {
6565     my ($vmid, $conf, $disk) = @_;
6566
6567     my $storecfg = PVE::Storage::config();
6568
6569     PVE::QemuConfig->foreach_volume($conf, sub {
6570         my ($ds, $drive) = @_;
6571
6572         return if drive_is_cdrom($drive);
6573         return if $disk && $ds ne $disk;
6574
6575         my $volid = $drive->{file};
6576         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6577
6578         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6579         $drive->{file} = $voliddst;
6580         $conf->{$ds} = print_drive($drive);
6581         PVE::QemuConfig->write_config($vmid, $conf);
6582     });
6583 }
6584
6585 sub convert_iscsi_path {
6586     my ($path) = @_;
6587
6588     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6589         my $portal = $1;
6590         my $target = $2;
6591         my $lun = $3;
6592
6593         my $initiator_name = get_initiator_name();
6594
6595         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6596                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6597     }
6598
6599     die "cannot convert iscsi path '$path', unkown format\n";
6600 }
6601
6602 sub qemu_img_convert {
6603     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6604
6605     my $storecfg = PVE::Storage::config();
6606     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6607     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6608
6609     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6610
6611     my $cachemode;
6612     my $src_path;
6613     my $src_is_iscsi = 0;
6614     my $src_format;
6615
6616     if ($src_storeid) {
6617         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6618         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6619         $src_format = qemu_img_format($src_scfg, $src_volname);
6620         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6621         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6622         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6623     } elsif (-f $src_volid) {
6624         $src_path = $src_volid;
6625         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6626             $src_format = $1;
6627         }
6628     }
6629
6630     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6631
6632     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6633     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6634     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6635     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6636
6637     my $cmd = [];
6638     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6639     push @$cmd, '-l', "snapshot.name=$snapname"
6640         if $snapname && $src_format && $src_format eq "qcow2";
6641     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6642     push @$cmd, '-T', $cachemode if defined($cachemode);
6643
6644     if ($src_is_iscsi) {
6645         push @$cmd, '--image-opts';
6646         $src_path = convert_iscsi_path($src_path);
6647     } elsif ($src_format) {
6648         push @$cmd, '-f', $src_format;
6649     }
6650
6651     if ($dst_is_iscsi) {
6652         push @$cmd, '--target-image-opts';
6653         $dst_path = convert_iscsi_path($dst_path);
6654     } else {
6655         push @$cmd, '-O', $dst_format;
6656     }
6657
6658     push @$cmd, $src_path;
6659
6660     if (!$dst_is_iscsi && $is_zero_initialized) {
6661         push @$cmd, "zeroinit:$dst_path";
6662     } else {
6663         push @$cmd, $dst_path;
6664     }
6665
6666     my $parser = sub {
6667         my $line = shift;
6668         if($line =~ m/\((\S+)\/100\%\)/){
6669             my $percent = $1;
6670             my $transferred = int($size * $percent / 100);
6671             my $remaining = $size - $transferred;
6672
6673             print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6674         }
6675
6676     };
6677
6678     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6679     my $err = $@;
6680     die "copy failed: $err" if $err;
6681 }
6682
6683 sub qemu_img_format {
6684     my ($scfg, $volname) = @_;
6685
6686     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6687         return $1;
6688     } else {
6689         return "raw";
6690     }
6691 }
6692
6693 sub qemu_drive_mirror {
6694     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
6695
6696     $jobs = {} if !$jobs;
6697
6698     my $qemu_target;
6699     my $format;
6700     $jobs->{"drive-$drive"} = {};
6701
6702     if ($dst_volid =~ /^nbd:/) {
6703         $qemu_target = $dst_volid;
6704         $format = "nbd";
6705     } else {
6706         my $storecfg = PVE::Storage::config();
6707         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6708
6709         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6710
6711         $format = qemu_img_format($dst_scfg, $dst_volname);
6712
6713         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6714
6715         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6716     }
6717
6718     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6719     $opts->{format} = $format if $format;
6720
6721     if (defined($src_bitmap)) {
6722         $opts->{sync} = 'incremental';
6723         $opts->{bitmap} = $src_bitmap;
6724         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
6725     }
6726
6727     if (defined($bwlimit)) {
6728         $opts->{speed} = $bwlimit * 1024;
6729         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6730     } else {
6731         print "drive mirror is starting for drive-$drive\n";
6732     }
6733
6734     # if a job already runs for this device we get an error, catch it for cleanup
6735     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
6736     if (my $err = $@) {
6737         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6738         warn "$@\n" if $@;
6739         die "mirroring error: $err\n";
6740     }
6741
6742     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
6743 }
6744
6745 # $completion can be either
6746 # 'complete': wait until all jobs are ready, block-job-complete them (default)
6747 # 'cancel': wait until all jobs are ready, block-job-cancel them
6748 # 'skip': wait until all jobs are ready, return with block jobs in ready state
6749 sub qemu_drive_mirror_monitor {
6750     my ($vmid, $vmiddst, $jobs, $completion, $qga) = @_;
6751
6752     $completion //= 'complete';
6753
6754     eval {
6755         my $err_complete = 0;
6756
6757         while (1) {
6758             die "storage migration timed out\n" if $err_complete > 300;
6759
6760             my $stats = mon_cmd($vmid, "query-block-jobs");
6761
6762             my $running_mirror_jobs = {};
6763             foreach my $stat (@$stats) {
6764                 next if $stat->{type} ne 'mirror';
6765                 $running_mirror_jobs->{$stat->{device}} = $stat;
6766             }
6767
6768             my $readycounter = 0;
6769
6770             foreach my $job (keys %$jobs) {
6771
6772                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6773                     print "$job : finished\n";
6774                     delete $jobs->{$job};
6775                     next;
6776                 }
6777
6778                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6779
6780                 my $busy = $running_mirror_jobs->{$job}->{busy};
6781                 my $ready = $running_mirror_jobs->{$job}->{ready};
6782                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6783                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6784                     my $remaining = $total - $transferred;
6785                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6786
6787                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6788                 }
6789
6790                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6791             }
6792
6793             last if scalar(keys %$jobs) == 0;
6794
6795             if ($readycounter == scalar(keys %$jobs)) {
6796                 print "all mirroring jobs are ready \n";
6797                 last if $completion eq 'skip'; #do the complete later
6798
6799                 if ($vmiddst && $vmiddst != $vmid) {
6800                     my $agent_running = $qga && qga_check_running($vmid);
6801                     if ($agent_running) {
6802                         print "freeze filesystem\n";
6803                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6804                     } else {
6805                         print "suspend vm\n";
6806                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6807                     }
6808
6809                     # if we clone a disk for a new target vm, we don't switch the disk
6810                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6811
6812                     if ($agent_running) {
6813                         print "unfreeze filesystem\n";
6814                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6815                     } else {
6816                         print "resume vm\n";
6817                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6818                     }
6819
6820                     last;
6821                 } else {
6822
6823                     foreach my $job (keys %$jobs) {
6824                         # try to switch the disk if source and destination are on the same guest
6825                         print "$job: Completing block job...\n";
6826
6827                         my $op;
6828                         if ($completion eq 'complete') {
6829                             $op = 'block-job-complete';
6830                         } elsif ($completion eq 'cancel') {
6831                             $op = 'block-job-cancel';
6832                         } else {
6833                             die "invalid completion value: $completion\n";
6834                         }
6835                         eval { mon_cmd($vmid, $op, device => $job) };
6836                         if ($@ =~ m/cannot be completed/) {
6837                             print "$job: Block job cannot be completed, try again.\n";
6838                             $err_complete++;
6839                         }else {
6840                             print "$job: Completed successfully.\n";
6841                             $jobs->{$job}->{complete} = 1;
6842                         }
6843                     }
6844                 }
6845             }
6846             sleep 1;
6847         }
6848     };
6849     my $err = $@;
6850
6851     if ($err) {
6852         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6853         die "mirroring error: $err";
6854     }
6855
6856 }
6857
6858 sub qemu_blockjobs_cancel {
6859     my ($vmid, $jobs) = @_;
6860
6861     foreach my $job (keys %$jobs) {
6862         print "$job: Cancelling block job\n";
6863         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
6864         $jobs->{$job}->{cancel} = 1;
6865     }
6866
6867     while (1) {
6868         my $stats = mon_cmd($vmid, "query-block-jobs");
6869
6870         my $running_jobs = {};
6871         foreach my $stat (@$stats) {
6872             $running_jobs->{$stat->{device}} = $stat;
6873         }
6874
6875         foreach my $job (keys %$jobs) {
6876
6877             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6878                 print "$job: Done.\n";
6879                 delete $jobs->{$job};
6880             }
6881         }
6882
6883         last if scalar(keys %$jobs) == 0;
6884
6885         sleep 1;
6886     }
6887 }
6888
6889 sub clone_disk {
6890     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6891         $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
6892
6893     my $newvolid;
6894
6895     if (!$full) {
6896         print "create linked clone of drive $drivename ($drive->{file})\n";
6897         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6898         push @$newvollist, $newvolid;
6899     } else {
6900
6901         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6902         $storeid = $storage if $storage;
6903
6904         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6905
6906         print "create full clone of drive $drivename ($drive->{file})\n";
6907         my $name = undef;
6908         my $size = undef;
6909         if (drive_is_cloudinit($drive)) {
6910             $name = "vm-$newvmid-cloudinit";
6911             $name .= ".$dst_format" if $dst_format ne 'raw';
6912             $snapname = undef;
6913             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
6914         } elsif ($drivename eq 'efidisk0') {
6915             $size = get_efivars_size($conf);
6916         } else {
6917             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6918         }
6919         $size /= 1024;
6920         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, $size);
6921         push @$newvollist, $newvolid;
6922
6923         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6924
6925         if (drive_is_cloudinit($drive)) {
6926             goto no_data_clone;
6927         }
6928
6929         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6930         if (!$running || $snapname) {
6931             # TODO: handle bwlimits
6932             if ($drivename eq 'efidisk0') {
6933                 # the relevant data on the efidisk may be smaller than the source
6934                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
6935                 # that is given by the OVMF_VARS.fd
6936                 my $src_path = PVE::Storage::path($storecfg, $drive->{file});
6937                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
6938                 run_command(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=1", "count=$size",
6939                     "if=$src_path", "of=$dst_path"]);
6940             } else {
6941                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6942             }
6943         } else {
6944
6945             my $kvmver = get_running_qemu_version ($vmid);
6946             if (!min_version($kvmver, 2, 7)) {
6947                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6948                     if $drive->{iothread};
6949             }
6950
6951             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs,
6952                 $completion, $qga, $bwlimit);
6953         }
6954     }
6955
6956 no_data_clone:
6957     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6958
6959     my $disk = $drive;
6960     $disk->{format} = undef;
6961     $disk->{file} = $newvolid;
6962     $disk->{size} = $size;
6963
6964     return $disk;
6965 }
6966
6967 sub get_running_qemu_version {
6968     my ($vmid) = @_;
6969     my $res = mon_cmd($vmid, "query-version");
6970     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6971 }
6972
6973 sub qemu_use_old_bios_files {
6974     my ($machine_type) = @_;
6975
6976     return if !$machine_type;
6977
6978     my $use_old_bios_files = undef;
6979
6980     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6981         $machine_type = $1;
6982         $use_old_bios_files = 1;
6983     } else {
6984         my $version = extract_version($machine_type, kvm_user_version());
6985         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6986         # load new efi bios files on migration. So this hack is required to allow
6987         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6988         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6989         $use_old_bios_files = !min_version($version, 2, 4);
6990     }
6991
6992     return ($use_old_bios_files, $machine_type);
6993 }
6994
6995 sub get_efivars_size {
6996     my ($conf) = @_;
6997     my $arch = get_vm_arch($conf);
6998     my (undef, $ovmf_vars) = get_ovmf_files($arch);
6999     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
7000     return -s $ovmf_vars;
7001 }
7002
7003 sub update_efidisk_size {
7004     my ($conf) = @_;
7005
7006     return if !defined($conf->{efidisk0});
7007
7008     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
7009     $disk->{size} = get_efivars_size($conf);
7010     $conf->{efidisk0} = print_drive($disk);
7011
7012     return;
7013 }
7014
7015 sub create_efidisk($$$$$) {
7016     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7017
7018     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7019     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7020
7021     my $vars_size_b = -s $ovmf_vars;
7022     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7023     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7024     PVE::Storage::activate_volumes($storecfg, [$volid]);
7025
7026     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7027     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7028
7029     return ($volid, $size/1024);
7030 }
7031
7032 sub vm_iothreads_list {
7033     my ($vmid) = @_;
7034
7035     my $res = mon_cmd($vmid, 'query-iothreads');
7036
7037     my $iothreads = {};
7038     foreach my $iothread (@$res) {
7039         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7040     }
7041
7042     return $iothreads;
7043 }
7044
7045 sub scsihw_infos {
7046     my ($conf, $drive) = @_;
7047
7048     my $maxdev = 0;
7049
7050     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7051         $maxdev = 7;
7052     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7053         $maxdev = 1;
7054     } else {
7055         $maxdev = 256;
7056     }
7057
7058     my $controller = int($drive->{index} / $maxdev);
7059     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
7060         ? "virtioscsi"
7061         : "scsihw";
7062
7063     return ($maxdev, $controller, $controller_prefix);
7064 }
7065
7066 sub windows_version {
7067     my ($ostype) = @_;
7068
7069     return 0 if !$ostype;
7070
7071     my $winversion = 0;
7072
7073     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7074         $winversion = 5;
7075     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7076         $winversion = 6;
7077     } elsif ($ostype =~ m/^win(\d+)$/) {
7078         $winversion = $1;
7079     }
7080
7081     return $winversion;
7082 }
7083
7084 sub resolve_dst_disk_format {
7085         my ($storecfg, $storeid, $src_volname, $format) = @_;
7086         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7087
7088         if (!$format) {
7089             # if no target format is specified, use the source disk format as hint
7090             if ($src_volname) {
7091                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7092                 $format = qemu_img_format($scfg, $src_volname);
7093             } else {
7094                 return $defFormat;
7095             }
7096         }
7097
7098         # test if requested format is supported - else use default
7099         my $supported = grep { $_ eq $format } @$validFormats;
7100         $format = $defFormat if !$supported;
7101         return $format;
7102 }
7103
7104 # NOTE: if this logic changes, please update docs & possibly gui logic
7105 sub find_vmstate_storage {
7106     my ($conf, $storecfg) = @_;
7107
7108     # first, return storage from conf if set
7109     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7110
7111     my ($target, $shared, $local);
7112
7113     foreach_storage_used_by_vm($conf, sub {
7114         my ($sid) = @_;
7115         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7116         my $dst = $scfg->{shared} ? \$shared : \$local;
7117         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7118     });
7119
7120     # second, use shared storage where VM has at least one disk
7121     # third, use local storage where VM has at least one disk
7122     # fall back to local storage
7123     $target = $shared // $local // 'local';
7124
7125     return $target;
7126 }
7127
7128 sub generate_uuid {
7129     my ($uuid, $uuid_str);
7130     UUID::generate($uuid);
7131     UUID::unparse($uuid, $uuid_str);
7132     return $uuid_str;
7133 }
7134
7135 sub generate_smbios1_uuid {
7136     return "uuid=".generate_uuid();
7137 }
7138
7139 sub nbd_stop {
7140     my ($vmid) = @_;
7141
7142     mon_cmd($vmid, 'nbd-server-stop');
7143 }
7144
7145 sub create_reboot_request {
7146     my ($vmid) = @_;
7147     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7148         or die "failed to create reboot trigger file: $!\n";
7149     close($fh);
7150 }
7151
7152 sub clear_reboot_request {
7153     my ($vmid) = @_;
7154     my $path = "/run/qemu-server/$vmid.reboot";
7155     my $res = 0;
7156
7157     $res = unlink($path);
7158     die "could not remove reboot request for $vmid: $!"
7159         if !$res && $! != POSIX::ENOENT;
7160
7161     return $res;
7162 }
7163
7164 sub bootorder_from_legacy {
7165     my ($conf, $bootcfg) = @_;
7166
7167     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
7168     my $bootindex_hash = {};
7169     my $i = 1;
7170     foreach my $o (split(//, $boot)) {
7171         $bootindex_hash->{$o} = $i*100;
7172         $i++;
7173     }
7174
7175     my $bootorder = {};
7176
7177     PVE::QemuConfig->foreach_volume($conf, sub {
7178         my ($ds, $drive) = @_;
7179
7180         if (drive_is_cdrom ($drive, 1)) {
7181             if ($bootindex_hash->{d}) {
7182                 $bootorder->{$ds} = $bootindex_hash->{d};
7183                 $bootindex_hash->{d} += 1;
7184             }
7185         } elsif ($bootindex_hash->{c}) {
7186             $bootorder->{$ds} = $bootindex_hash->{c}
7187                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
7188             $bootindex_hash->{c} += 1;
7189         }
7190     });
7191
7192     if ($bootindex_hash->{n}) {
7193         for (my $i = 0; $i < $MAX_NETS; $i++) {
7194             my $netname = "net$i";
7195             next if !$conf->{$netname};
7196             $bootorder->{$netname} = $bootindex_hash->{n};
7197             $bootindex_hash->{n} += 1;
7198         }
7199     }
7200
7201     return $bootorder;
7202 }
7203
7204 # Generate default device list for 'boot: order=' property. Matches legacy
7205 # default boot order, but with explicit device names. This is important, since
7206 # the fallback for when neither 'order' nor the old format is specified relies
7207 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
7208 sub get_default_bootdevices {
7209     my ($conf) = @_;
7210
7211     my @ret = ();
7212
7213     # harddisk
7214     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
7215     push @ret, $first if $first;
7216
7217     # cdrom
7218     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
7219     push @ret, $first if $first;
7220
7221     # network
7222     for (my $i = 0; $i < $MAX_NETS; $i++) {
7223         my $netname = "net$i";
7224         next if !$conf->{$netname};
7225         push @ret, $netname;
7226         last;
7227     }
7228
7229     return \@ret;
7230 }
7231
7232 sub device_bootorder {
7233     my ($conf) = @_;
7234
7235     return bootorder_from_legacy($conf) if !defined($conf->{boot});
7236
7237     my $boot = parse_property_string($boot_fmt, $conf->{boot});
7238
7239     my $bootorder = {};
7240     if (!defined($boot) || $boot->{legacy}) {
7241         $bootorder = bootorder_from_legacy($conf, $boot);
7242     } elsif ($boot->{order}) {
7243         my $i = 100; # start at 100 to allow user to insert devices before us with -args
7244         for my $dev (PVE::Tools::split_list($boot->{order})) {
7245             $bootorder->{$dev} = $i++;
7246         }
7247     }
7248
7249     return $bootorder;
7250 }
7251
7252 # bash completion helper
7253
7254 sub complete_backup_archives {
7255     my ($cmdname, $pname, $cvalue) = @_;
7256
7257     my $cfg = PVE::Storage::config();
7258
7259     my $storeid;
7260
7261     if ($cvalue =~ m/^([^:]+):/) {
7262         $storeid = $1;
7263     }
7264
7265     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7266
7267     my $res = [];
7268     foreach my $id (keys %$data) {
7269         foreach my $item (@{$data->{$id}}) {
7270             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
7271             push @$res, $item->{volid} if defined($item->{volid});
7272         }
7273     }
7274
7275     return $res;
7276 }
7277
7278 my $complete_vmid_full = sub {
7279     my ($running) = @_;
7280
7281     my $idlist = vmstatus();
7282
7283     my $res = [];
7284
7285     foreach my $id (keys %$idlist) {
7286         my $d = $idlist->{$id};
7287         if (defined($running)) {
7288             next if $d->{template};
7289             next if $running && $d->{status} ne 'running';
7290             next if !$running && $d->{status} eq 'running';
7291         }
7292         push @$res, $id;
7293
7294     }
7295     return $res;
7296 };
7297
7298 sub complete_vmid {
7299     return &$complete_vmid_full();
7300 }
7301
7302 sub complete_vmid_stopped {
7303     return &$complete_vmid_full(0);
7304 }
7305
7306 sub complete_vmid_running {
7307     return &$complete_vmid_full(1);
7308 }
7309
7310 sub complete_storage {
7311
7312     my $cfg = PVE::Storage::config();
7313     my $ids = $cfg->{ids};
7314
7315     my $res = [];
7316     foreach my $sid (keys %$ids) {
7317         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7318         next if !$ids->{$sid}->{content}->{images};
7319         push @$res, $sid;
7320     }
7321
7322     return $res;
7323 }
7324
7325 sub complete_migration_storage {
7326     my ($cmd, $param, $current_value, $all_args) = @_;
7327
7328     my $targetnode = @$all_args[1];
7329
7330     my $cfg = PVE::Storage::config();
7331     my $ids = $cfg->{ids};
7332
7333     my $res = [];
7334     foreach my $sid (keys %$ids) {
7335         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7336         next if !$ids->{$sid}->{content}->{images};
7337         push @$res, $sid;
7338     }
7339
7340     return $res;
7341 }
7342
7343 1;