PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::DataCenterConfig;
  31 use PVE::Exception qw(raise raise_param_exc);
  32 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  33 use PVE::INotify;
  34 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  35 use PVE::ProcFSTools;
  36 use PVE::RPCEnvironment;
  37 use PVE::Storage;
  38 use PVE::SysFSTools;
  39 use PVE::Systemd;
  40 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  41
  42 use PVE::QMPClient;
  43 use PVE::QemuConfig;
  44 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  45 use PVE::QemuServer::Cloudinit;
  46 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  47 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom parse_drive print_drive);
  48 use PVE::QemuServer::Machine;
  49 use PVE::QemuServer::Memory;
  50 use PVE::QemuServer::Monitor qw(mon_cmd);
  51 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  52 use PVE::QemuServer::USB qw(parse_usb_device);
  53
  54 my $have_sdn;
  55 eval {
  56     require PVE::Network::SDN::Zones;
  57     $have_sdn = 1;
  58 };
  59
  60 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  61 my $OVMF = {
  62     x86_64 => [
  63         "$EDK2_FW_BASE/OVMF_CODE.fd",
  64         "$EDK2_FW_BASE/OVMF_VARS.fd"
  65     ],
  66     aarch64 => [
  67         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  68         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  69     ],
  70 };
  71
  72 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  73
  74 # Note about locking: we use flock on the config file protect
  75 # against concurent actions.
  76 # Aditionaly, we have a 'lock' setting in the config file. This
  77 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  78 # allowed when such lock is set. But you can ignore this kind of
  79 # lock with the --skiplock flag.
  80
  81 cfs_register_file('/qemu-server/',
  82                   \&parse_vm_config,
  83                   \&write_vm_config);
  84
  85 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  86     description => "Some command save/restore state from this location.",
  87     type => 'string',
  88     maxLength => 128,
  89     optional => 1,
  90 });
  91
  92 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  93         description => "Specifies the Qemu machine type.",
  94         type => 'string',
  95         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
  96         maxLength => 40,
  97         optional => 1,
  98 });
  99
 100
 101 sub map_storage {
 102     my ($map, $source) = @_;
 103
 104     return $source if !defined($map);
 105
 106     return $map->{entries}->{$source}
 107         if $map->{entries} && defined($map->{entries}->{$source});
 108
 109     return $map->{default} if $map->{default};
 110
 111     # identity (fallback)
 112     return $source;
 113 }
 114
 115 PVE::JSONSchema::register_standard_option('pve-targetstorage', {
 116     description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
 117     type => 'string',
 118     format => 'storagepair-list',
 119     optional => 1,
 120 });
 121
 122 #no warnings 'redefine';
 123
 124 sub cgroups_write {
 125    my ($controller, $vmid, $option, $value) = @_;
 126
 127    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 128    PVE::ProcFSTools::write_proc_entry($path, $value);
 129
 130 }
 131
 132 my $nodename_cache;
 133 sub nodename {
 134     $nodename_cache //= PVE::INotify::nodename();
 135     return $nodename_cache;
 136 }
 137
 138 my $watchdog_fmt = {
 139     model => {
 140         default_key => 1,
 141         type => 'string',
 142         enum => [qw(i6300esb ib700)],
 143         description => "Watchdog type to emulate.",
 144         default => 'i6300esb',
 145         optional => 1,
 146     },
 147     action => {
 148         type => 'string',
 149         enum => [qw(reset shutdown poweroff pause debug none)],
 150         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 151         optional => 1,
 152     },
 153 };
 154 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 155
 156 my $agent_fmt = {
 157     enabled => {
 158         description => "Enable/disable Qemu GuestAgent.",
 159         type => 'boolean',
 160         default => 0,
 161         default_key => 1,
 162     },
 163     fstrim_cloned_disks => {
 164         description => "Run fstrim after cloning/moving a disk.",
 165         type => 'boolean',
 166         optional => 1,
 167         default => 0
 168     },
 169     type => {
 170         description => "Select the agent type",
 171         type => 'string',
 172         default => 'virtio',
 173         optional => 1,
 174         enum => [qw(virtio isa)],
 175     },
 176 };
 177
 178 my $vga_fmt = {
 179     type => {
 180         description => "Select the VGA type.",
 181         type => 'string',
 182         default => 'std',
 183         optional => 1,
 184         default_key => 1,
 185         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 186     },
 187     memory => {
 188         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 189         type => 'integer',
 190         optional => 1,
 191         minimum => 4,
 192         maximum => 512,
 193     },
 194 };
 195
 196 my $ivshmem_fmt = {
 197     size => {
 198         type => 'integer',
 199         minimum => 1,
 200         description => "The size of the file in MB.",
 201     },
 202     name => {
 203         type => 'string',
 204         pattern => '[a-zA-Z0-9\-]+',
 205         optional => 1,
 206         format_description => 'string',
 207         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 208     },
 209 };
 210
 211 my $audio_fmt = {
 212     device => {
 213         type => 'string',
 214         enum => [qw(ich9-intel-hda intel-hda AC97)],
 215         description =>  "Configure an audio device."
 216     },
 217     driver =>  {
 218         type => 'string',
 219         enum => ['spice'],
 220         default => 'spice',
 221         optional => 1,
 222         description => "Driver backend for the audio device."
 223     },
 224 };
 225
 226 my $spice_enhancements_fmt = {
 227     foldersharing => {
 228         type => 'boolean',
 229         optional => 1,
 230         default => '0',
 231         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 232     },
 233     videostreaming =>  {
 234         type => 'string',
 235         enum => ['off', 'all', 'filter'],
 236         default => 'off',
 237         optional => 1,
 238         description => "Enable video streaming. Uses compression for detected video streams."
 239     },
 240 };
 241
 242 my $rng_fmt = {
 243     source => {
 244         type => 'string',
 245         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 246         default_key => 1,
 247         description => "The file on the host to gather entropy from. In most"
 248                      . " cases /dev/urandom should be preferred over /dev/random"
 249                      . " to avoid entropy-starvation issues on the host. Using"
 250                      . " urandom does *not* decrease security in any meaningful"
 251                      . " way, as it's still seeded from real entropy, and the"
 252                      . " bytes provided will most likely be mixed with real"
 253                      . " entropy on the guest as well. /dev/hwrng can be used"
 254                      . " to pass through a hardware RNG from the host.",
 255     },
 256     max_bytes => {
 257         type => 'integer',
 258         description => "Maximum bytes of entropy injected into the guest every"
 259                      . " 'period' milliseconds. Prefer a lower value when using"
 260                      . " /dev/random as source. Use 0 to disable limiting"
 261                      . " (potentially dangerous!).",
 262         optional => 1,
 263
 264         # default is 1 KiB/s, provides enough entropy to the guest to avoid
 265         # boot-starvation issues (e.g. systemd etc...) while allowing no chance
 266         # of overwhelming the host, provided we're reading from /dev/urandom
 267         default => 1024,
 268     },
 269     period => {
 270         type => 'integer',
 271         description => "Every 'period' milliseconds the entropy-injection quota"
 272                      . " is reset, allowing the guest to retrieve another"
 273                      . " 'max_bytes' of entropy.",
 274         optional => 1,
 275         default => 1000,
 276     },
 277 };
 278
 279 my $confdesc = {
 280     onboot => {
 281         optional => 1,
 282         type => 'boolean',
 283         description => "Specifies whether a VM will be started during system bootup.",
 284         default => 0,
 285     },
 286     autostart => {
 287         optional => 1,
 288         type => 'boolean',
 289         description => "Automatic restart after crash (currently ignored).",
 290         default => 0,
 291     },
 292     hotplug => {
 293         optional => 1,
 294         type => 'string', format => 'pve-hotplug-features',
 295         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 296         default => 'network,disk,usb',
 297     },
 298     reboot => {
 299         optional => 1,
 300         type => 'boolean',
 301         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 302         default => 1,
 303     },
 304     lock => {
 305         optional => 1,
 306         type => 'string',
 307         description => "Lock/unlock the VM.",
 308         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 309     },
 310     cpulimit => {
 311         optional => 1,
 312         type => 'number',
 313         description => "Limit of CPU usage.",
 314         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 315         minimum => 0,
 316         maximum => 128,
 317         default => 0,
 318     },
 319     cpuunits => {
 320         optional => 1,
 321         type => 'integer',
 322         description => "CPU weight for a VM.",
 323         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 324         minimum => 2,
 325         maximum => 262144,
 326         default => 1024,
 327     },
 328     memory => {
 329         optional => 1,
 330         type => 'integer',
 331         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 332         minimum => 16,
 333         default => 512,
 334     },
 335     balloon => {
 336         optional => 1,
 337         type => 'integer',
 338         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 339         minimum => 0,
 340     },
 341     shares => {
 342         optional => 1,
 343         type => 'integer',
 344         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 345         minimum => 0,
 346         maximum => 50000,
 347         default => 1000,
 348     },
 349     keyboard => {
 350         optional => 1,
 351         type => 'string',
 352         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 353                        "It should not be necessary to set it.",
 354         enum => PVE::Tools::kvmkeymaplist(),
 355         default => undef,
 356     },
 357     name => {
 358         optional => 1,
 359         type => 'string', format => 'dns-name',
 360         description => "Set a name for the VM. Only used on the configuration web interface.",
 361     },
 362     scsihw => {
 363         optional => 1,
 364         type => 'string',
 365         description => "SCSI controller model",
 366         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 367         default => 'lsi',
 368     },
 369     description => {
 370         optional => 1,
 371         type => 'string',
 372         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 373     },
 374     ostype => {
 375         optional => 1,
 376         type => 'string',
 377         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 378         description => "Specify guest operating system.",
 379         verbose_description => <<EODESC,
 380 Specify guest operating system. This is used to enable special
 381 optimization/features for specific operating systems:
 382
 383 [horizontal]
 384 other;; unspecified OS
 385 wxp;; Microsoft Windows XP
 386 w2k;; Microsoft Windows 2000
 387 w2k3;; Microsoft Windows 2003
 388 w2k8;; Microsoft Windows 2008
 389 wvista;; Microsoft Windows Vista
 390 win7;; Microsoft Windows 7
 391 win8;; Microsoft Windows 8/2012/2012r2
 392 win10;; Microsoft Windows 10/2016
 393 l24;; Linux 2.4 Kernel
 394 l26;; Linux 2.6 - 5.X Kernel
 395 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 396 EODESC
 397     },
 398     boot => {
 399         optional => 1,
 400         type => 'string', format => 'pve-qm-boot',
 401         description => "Specify guest boot order. Use with 'order=', usage with"
 402                      . " no key or 'legacy=' is deprecated.",
 403     },
 404     bootdisk => {
 405         optional => 1,
 406         type => 'string', format => 'pve-qm-bootdisk',
 407         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 408         pattern => '(ide|sata|scsi|virtio)\d+',
 409     },
 410     smp => {
 411         optional => 1,
 412         type => 'integer',
 413         description => "The number of CPUs. Please use option -sockets instead.",
 414         minimum => 1,
 415         default => 1,
 416     },
 417     sockets => {
 418         optional => 1,
 419         type => 'integer',
 420         description => "The number of CPU sockets.",
 421         minimum => 1,
 422         default => 1,
 423     },
 424     cores => {
 425         optional => 1,
 426         type => 'integer',
 427         description => "The number of cores per socket.",
 428         minimum => 1,
 429         default => 1,
 430     },
 431     numa => {
 432         optional => 1,
 433         type => 'boolean',
 434         description => "Enable/disable NUMA.",
 435         default => 0,
 436     },
 437     hugepages => {
 438         optional => 1,
 439         type => 'string',
 440         description => "Enable/disable hugepages memory.",
 441         enum => [qw(any 2 1024)],
 442     },
 443     keephugepages => {
 444         optional => 1,
 445         type => 'boolean',
 446         default => 0,
 447         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 448             ." after VM shutdown and can be used for subsequent starts.",
 449     },
 450     vcpus => {
 451         optional => 1,
 452         type => 'integer',
 453         description => "Number of hotplugged vcpus.",
 454         minimum => 1,
 455         default => 0,
 456     },
 457     acpi => {
 458         optional => 1,
 459         type => 'boolean',
 460         description => "Enable/disable ACPI.",
 461         default => 1,
 462     },
 463     agent => {
 464         optional => 1,
 465         description => "Enable/disable Qemu GuestAgent and its properties.",
 466         type => 'string',
 467         format => $agent_fmt,
 468     },
 469     kvm => {
 470         optional => 1,
 471         type => 'boolean',
 472         description => "Enable/disable KVM hardware virtualization.",
 473         default => 1,
 474     },
 475     tdf => {
 476         optional => 1,
 477         type => 'boolean',
 478         description => "Enable/disable time drift fix.",
 479         default => 0,
 480     },
 481     localtime => {
 482         optional => 1,
 483         type => 'boolean',
 484         description => "Set the real time clock to local time. This is enabled by default if ostype"
 485             ." indicates a Microsoft OS.",
 486     },
 487     freeze => {
 488         optional => 1,
 489         type => 'boolean',
 490         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 491     },
 492     vga => {
 493         optional => 1,
 494         type => 'string', format => $vga_fmt,
 495         description => "Configure the VGA hardware.",
 496         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 497             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 498             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 499             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 500             ." display server. For win* OS you can select how many independent displays you want,"
 501             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 502             ." using a serial device as terminal.",
 503     },
 504     watchdog => {
 505         optional => 1,
 506         type => 'string', format => 'pve-qm-watchdog',
 507         description => "Create a virtual hardware watchdog device.",
 508         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 509             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 510             ." else the watchdog will reset the guest (or execute the respective action specified)",
 511     },
 512     startdate => {
 513         optional => 1,
 514         type => 'string',
 515         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 516         description => "Set the initial date of the real time clock. Valid format for date are:"
 517             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 518         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 519         default => 'now',
 520     },
 521     startup =>  get_standard_option('pve-startup-order'),
 522     template => {
 523         optional => 1,
 524         type => 'boolean',
 525         description => "Enable/disable Template.",
 526         default => 0,
 527     },
 528     args => {
 529         optional => 1,
 530         type => 'string',
 531         description => "Arbitrary arguments passed to kvm.",
 532         verbose_description => <<EODESCR,
 533 Arbitrary arguments passed to kvm, for example:
 534
 535 args: -no-reboot -no-hpet
 536
 537 NOTE: this option is for experts only.
 538 EODESCR
 539     },
 540     tablet => {
 541         optional => 1,
 542         type => 'boolean',
 543         default => 1,
 544         description => "Enable/disable the USB tablet device.",
 545         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 546             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 547             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 548             ." may consider disabling this to save some context switches. This is turned off by"
 549             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 550     },
 551     migrate_speed => {
 552         optional => 1,
 553         type => 'integer',
 554         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 555         minimum => 0,
 556         default => 0,
 557     },
 558     migrate_downtime => {
 559         optional => 1,
 560         type => 'number',
 561         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 562         minimum => 0,
 563         default => 0.1,
 564     },
 565     cdrom => {
 566         optional => 1,
 567         type => 'string', format => 'pve-qm-ide',
 568         typetext => '<volume>',
 569         description => "This is an alias for option -ide2",
 570     },
 571     cpu => {
 572         optional => 1,
 573         description => "Emulated CPU type.",
 574         type => 'string',
 575         format => 'pve-vm-cpu-conf',
 576     },
 577     parent => get_standard_option('pve-snapshot-name', {
 578         optional => 1,
 579         description => "Parent snapshot name. This is used internally, and should not be modified.",
 580     }),
 581     snaptime => {
 582         optional => 1,
 583         description => "Timestamp for snapshots.",
 584         type => 'integer',
 585         minimum => 0,
 586     },
 587     vmstate => {
 588         optional => 1,
 589         type => 'string', format => 'pve-volume-id',
 590         description => "Reference to a volume which stores the VM state. This is used internally"
 591             ." for snapshots.",
 592     },
 593     vmstatestorage => get_standard_option('pve-storage-id', {
 594         description => "Default storage for VM state volumes/files.",
 595         optional => 1,
 596     }),
 597     runningmachine => get_standard_option('pve-qemu-machine', {
 598         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 599             ." for snapshots.",
 600     }),
 601     runningcpu => {
 602         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 603             ." internally for snapshots.",
 604         optional => 1,
 605         type => 'string',
 606         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 607         format_description => 'QEMU -cpu parameter'
 608     },
 609     machine => get_standard_option('pve-qemu-machine'),
 610     arch => {
 611         description => "Virtual processor architecture. Defaults to the host.",
 612         optional => 1,
 613         type => 'string',
 614         enum => [qw(x86_64 aarch64)],
 615     },
 616     smbios1 => {
 617         description => "Specify SMBIOS type 1 fields.",
 618         type => 'string', format => 'pve-qm-smbios1',
 619         maxLength => 512,
 620         optional => 1,
 621     },
 622     protection => {
 623         optional => 1,
 624         type => 'boolean',
 625         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 626             ." remove disk operations.",
 627         default => 0,
 628     },
 629     bios => {
 630         optional => 1,
 631         type => 'string',
 632         enum => [ qw(seabios ovmf) ],
 633         description => "Select BIOS implementation.",
 634         default => 'seabios',
 635     },
 636     vmgenid => {
 637         type => 'string',
 638         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 639         format_description => 'UUID',
 640         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 641             ." to disable explicitly.",
 642         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 643             ." value identifier to the guest OS. This allows to notify the guest operating system"
 644             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 645             ." execution or creation from a template). The guest operating system notices the"
 646             ." change, and is then able to react as appropriate by marking its copies of"
 647             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 648             ."Note that auto-creation only works when done through API/CLI create or update methods"
 649             .", but not when manually editing the config file.",
 650         default => "1 (autogenerated)",
 651         optional => 1,
 652     },
 653     hookscript => {
 654         type => 'string',
 655         format => 'pve-volume-id',
 656         optional => 1,
 657         description => "Script that will be executed during various steps in the vms lifetime.",
 658     },
 659     ivshmem => {
 660         type => 'string',
 661         format => $ivshmem_fmt,
 662         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 663             ." the host.",
 664         optional => 1,
 665     },
 666     audio0 => {
 667         type => 'string',
 668         format => $audio_fmt,
 669         description => "Configure a audio device, useful in combination with QXL/Spice.",
 670         optional => 1
 671     },
 672     spice_enhancements => {
 673         type => 'string',
 674         format => $spice_enhancements_fmt,
 675         description => "Configure additional enhancements for SPICE.",
 676         optional => 1
 677     },
 678     tags => {
 679         type => 'string', format => 'pve-tag-list',
 680         description => 'Tags of the VM. This is only meta information.',
 681         optional => 1,
 682     },
 683     rng0 => {
 684         type => 'string',
 685         format => $rng_fmt,
 686         description => "Configure a VirtIO-based Random Number Generator.",
 687         optional => 1,
 688     },
 689 };
 690
 691 my $cicustom_fmt = {
 692     meta => {
 693         type => 'string',
 694         optional => 1,
 695         description => 'Specify a custom file containing all meta data passed to the VM via"
 696             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 697         format => 'pve-volume-id',
 698         format_description => 'volume',
 699     },
 700     network => {
 701         type => 'string',
 702         optional => 1,
 703         description => 'Specify a custom file containing all network data passed to the VM via'
 704             .' cloud-init.',
 705         format => 'pve-volume-id',
 706         format_description => 'volume',
 707     },
 708     user => {
 709         type => 'string',
 710         optional => 1,
 711         description => 'Specify a custom file containing all user data passed to the VM via'
 712             .' cloud-init.',
 713         format => 'pve-volume-id',
 714         format_description => 'volume',
 715     },
 716 };
 717 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 718
 719 my $confdesc_cloudinit = {
 720     citype => {
 721         optional => 1,
 722         type => 'string',
 723         description => 'Specifies the cloud-init configuration format. The default depends on the'
 724             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 725             .' and `configdrive2` for windows.',
 726         enum => ['configdrive2', 'nocloud'],
 727     },
 728     ciuser => {
 729         optional => 1,
 730         type => 'string',
 731         description => "cloud-init: User name to change ssh keys and password for instead of the"
 732             ." image's configured default user.",
 733     },
 734     cipassword => {
 735         optional => 1,
 736         type => 'string',
 737         description => 'cloud-init: Password to assign the user. Using this is generally not'
 738             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 739             .' support hashed passwords.',
 740     },
 741     cicustom => {
 742         optional => 1,
 743         type => 'string',
 744         description => 'cloud-init: Specify custom files to replace the automatically generated'
 745             .' ones at start.',
 746         format => 'pve-qm-cicustom',
 747     },
 748     searchdomain => {
 749         optional => 1,
 750         type => 'string',
 751         description => "cloud-init: Sets DNS search domains for a container. Create will'
 752             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 753             .' are set.",
 754     },
 755     nameserver => {
 756         optional => 1,
 757         type => 'string', format => 'address-list',
 758         description => "cloud-init: Sets DNS server IP address for a container. Create will'
 759             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 760             .' are set.",
 761     },
 762     sshkeys => {
 763         optional => 1,
 764         type => 'string',
 765         format => 'urlencoded',
 766         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 767     },
 768 };
 769
 770 # what about other qemu settings ?
 771 #cpu => 'string',
 772 #machine => 'string',
 773 #fda => 'file',
 774 #fdb => 'file',
 775 #mtdblock => 'file',
 776 #sd => 'file',
 777 #pflash => 'file',
 778 #snapshot => 'bool',
 779 #bootp => 'file',
 780 ##tftp => 'dir',
 781 ##smb => 'dir',
 782 #kernel => 'file',
 783 #append => 'string',
 784 #initrd => 'file',
 785 ##soundhw => 'string',
 786
 787 while (my ($k, $v) = each %$confdesc) {
 788     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 789 }
 790
 791 my $MAX_USB_DEVICES = 5;
 792 my $MAX_NETS = 32;
 793 my $MAX_SERIAL_PORTS = 4;
 794 my $MAX_PARALLEL_PORTS = 3;
 795 my $MAX_NUMA = 8;
 796
 797 my $numa_fmt = {
 798     cpus => {
 799         type => "string",
 800         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 801         description => "CPUs accessing this NUMA node.",
 802         format_description => "id[-id];...",
 803     },
 804     memory => {
 805         type => "number",
 806         description => "Amount of memory this NUMA node provides.",
 807         optional => 1,
 808     },
 809     hostnodes => {
 810         type => "string",
 811         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 812         description => "Host NUMA nodes to use.",
 813         format_description => "id[-id];...",
 814         optional => 1,
 815     },
 816     policy => {
 817         type => 'string',
 818         enum => [qw(preferred bind interleave)],
 819         description => "NUMA allocation policy.",
 820         optional => 1,
 821     },
 822 };
 823 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 824 my $numadesc = {
 825     optional => 1,
 826     type => 'string', format => $numa_fmt,
 827     description => "NUMA topology.",
 828 };
 829 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 830
 831 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 832     $confdesc->{"numa$i"} = $numadesc;
 833 }
 834
 835 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 836                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 837                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 838 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 839
 840 my $net_fmt_bridge_descr = <<__EOD__;
 841 Bridge to attach the network device to. The Proxmox VE standard bridge
 842 is called 'vmbr0'.
 843
 844 If you do not specify a bridge, we create a kvm user (NATed) network
 845 device, which provides DHCP and DNS services. The following addresses
 846 are used:
 847
 848  10.0.2.2   Gateway
 849  10.0.2.3   DNS Server
 850  10.0.2.4   SMB Server
 851
 852 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 853 __EOD__
 854
 855 my $net_fmt = {
 856     macaddr  => get_standard_option('mac-addr', {
 857         description => "MAC address. That address must be unique withing your network. This is"
 858             ." automatically generated if not specified.",
 859     }),
 860     model => {
 861         type => 'string',
 862         description => "Network Card Model. The 'virtio' model provides the best performance with"
 863             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 864             ." best to use 'e1000'.",
 865         enum => $nic_model_list,
 866         default_key => 1,
 867     },
 868     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 869     bridge => {
 870         type => 'string',
 871         description => $net_fmt_bridge_descr,
 872         format_description => 'bridge',
 873         pattern => '[-_.\w\d]+',
 874         optional => 1,
 875     },
 876     queues => {
 877         type => 'integer',
 878         minimum => 0, maximum => 16,
 879         description => 'Number of packet queues to be used on the device.',
 880         optional => 1,
 881     },
 882     rate => {
 883         type => 'number',
 884         minimum => 0,
 885         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 886         optional => 1,
 887     },
 888     tag => {
 889         type => 'integer',
 890         minimum => 1, maximum => 4094,
 891         description => 'VLAN tag to apply to packets on this interface.',
 892         optional => 1,
 893     },
 894     trunks => {
 895         type => 'string',
 896         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 897         description => 'VLAN trunks to pass through this interface.',
 898         format_description => 'vlanid[;vlanid...]',
 899         optional => 1,
 900     },
 901     firewall => {
 902         type => 'boolean',
 903         description => 'Whether this interface should be protected by the firewall.',
 904         optional => 1,
 905     },
 906     link_down => {
 907         type => 'boolean',
 908         description => 'Whether this interface should be disconnected (like pulling the plug).',
 909         optional => 1,
 910     },
 911     mtu => {
 912         type => 'integer',
 913         minimum => 1, maximum => 65520,
 914         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 915         optional => 1,
 916     },
 917 };
 918
 919 my $netdesc = {
 920     optional => 1,
 921     type => 'string', format => $net_fmt,
 922     description => "Specify network devices.",
 923 };
 924
 925 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 926
 927 my $ipconfig_fmt = {
 928     ip => {
 929         type => 'string',
 930         format => 'pve-ipv4-config',
 931         format_description => 'IPv4Format/CIDR',
 932         description => 'IPv4 address in CIDR format.',
 933         optional => 1,
 934         default => 'dhcp',
 935     },
 936     gw => {
 937         type => 'string',
 938         format => 'ipv4',
 939         format_description => 'GatewayIPv4',
 940         description => 'Default gateway for IPv4 traffic.',
 941         optional => 1,
 942         requires => 'ip',
 943     },
 944     ip6 => {
 945         type => 'string',
 946         format => 'pve-ipv6-config',
 947         format_description => 'IPv6Format/CIDR',
 948         description => 'IPv6 address in CIDR format.',
 949         optional => 1,
 950         default => 'dhcp',
 951     },
 952     gw6 => {
 953         type => 'string',
 954         format => 'ipv6',
 955         format_description => 'GatewayIPv6',
 956         description => 'Default gateway for IPv6 traffic.',
 957         optional => 1,
 958         requires => 'ip6',
 959     },
 960 };
 961 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 962 my $ipconfigdesc = {
 963     optional => 1,
 964     type => 'string', format => 'pve-qm-ipconfig',
 965     description => <<'EODESCR',
 966 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 967
 968 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 969
 970 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
 971 gateway should be provided.
 972 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 973
 974 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
 975 dhcp on IPv4.
 976 EODESCR
 977 };
 978 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 979
 980 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 981     $confdesc->{"net$i"} = $netdesc;
 982     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 983 }
 984
 985 foreach my $key (keys %$confdesc_cloudinit) {
 986     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 987 }
 988
 989 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 990 sub verify_volume_id_or_qm_path {
 991     my ($volid, $noerr) = @_;
 992
 993     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 994         return $volid;
 995     }
 996
 997     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 998     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 999     if ($@) {
1000         return if $noerr;
1001         die $@;
1002     }
1003     return $volid;
1004 }
1005
1006 my $usb_fmt = {
1007     host => {
1008         default_key => 1,
1009         type => 'string', format => 'pve-qm-usb-device',
1010         format_description => 'HOSTUSBDEVICE|spice',
1011         description => <<EODESCR,
1012 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1013
1014  'bus-port(.port)*' (decimal numbers) or
1015  'vendor_id:product_id' (hexadeciaml numbers) or
1016  'spice'
1017
1018 You can use the 'lsusb -t' command to list existing usb devices.
1019
1020 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1021 machines - use with special care.
1022
1023 The value 'spice' can be used to add a usb redirection devices for spice.
1024 EODESCR
1025     },
1026     usb3 => {
1027         optional => 1,
1028         type => 'boolean',
1029         description => "Specifies whether if given host option is a USB3 device or port.",
1030         default => 0,
1031     },
1032 };
1033
1034 my $usbdesc = {
1035     optional => 1,
1036     type => 'string', format => $usb_fmt,
1037     description => "Configure an USB device (n is 0 to 4).",
1038 };
1039 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1040
1041 my $serialdesc = {
1042         optional => 1,
1043         type => 'string',
1044         pattern => '(/dev/.+|socket)',
1045         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1046         verbose_description =>  <<EODESCR,
1047 Create a serial device inside the VM (n is 0 to 3), and pass through a
1048 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1049 host side (use 'qm terminal' to open a terminal connection).
1050
1051 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1052 use with special care.
1053
1054 CAUTION: Experimental! User reported problems with this option.
1055 EODESCR
1056 };
1057
1058 my $paralleldesc= {
1059         optional => 1,
1060         type => 'string',
1061         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1062         description =>  "Map host parallel devices (n is 0 to 2).",
1063         verbose_description =>  <<EODESCR,
1064 Map host parallel devices (n is 0 to 2).
1065
1066 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1067 machines - use with special care.
1068
1069 CAUTION: Experimental! User reported problems with this option.
1070 EODESCR
1071 };
1072
1073 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1074     $confdesc->{"parallel$i"} = $paralleldesc;
1075 }
1076
1077 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1078     $confdesc->{"serial$i"} = $serialdesc;
1079 }
1080
1081 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1082     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1083 }
1084
1085 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1086     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1087 }
1088
1089 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1090     $confdesc->{"usb$i"} = $usbdesc;
1091 }
1092
1093 my $boot_fmt = {
1094     legacy => {
1095         optional => 1,
1096         default_key => 1,
1097         type => 'string',
1098         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1099                      . " Deprecated, use 'order=' instead.",
1100         pattern => '[acdn]{1,4}',
1101         format_description => "[acdn]{1,4}",
1102
1103         # note: this is also the fallback if boot: is not given at all
1104         default => 'cdn',
1105     },
1106     order => {
1107         optional => 1,
1108         type => 'string',
1109         format => 'pve-qm-bootdev-list',
1110         format_description => "device[;device...]",
1111         description => <<EODESC,
1112 The guest will attempt to boot from devices in the order they appear here.
1113
1114 Disks, optical drives and passed-through storage USB devices will be directly
1115 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1116 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1117
1118 Note that only devices in this list will be marked as bootable and thus loaded
1119 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1120 (e.g. software-raid), you need to specify all of them here.
1121
1122 Overrides the deprecated 'legacy=[acdn]*' value when given.
1123 EODESC
1124     },
1125 };
1126 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1127
1128 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1129 sub verify_bootdev {
1130     my ($dev, $noerr) = @_;
1131
1132     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && $dev !~ m/^efidisk/;
1133
1134     my $check = sub {
1135         my ($base) = @_;
1136         return 0 if $dev !~ m/^$base\d+$/;
1137         return 0 if !$confdesc->{$dev};
1138         return 1;
1139     };
1140
1141     return $dev if $check->("net");
1142     return $dev if $check->("usb");
1143     return $dev if $check->("hostpci");
1144
1145     return if $noerr;
1146     die "invalid boot device '$dev'\n";
1147 }
1148
1149 sub print_bootorder {
1150     my ($devs) = @_;
1151     my $data = { order => join(';', @$devs) };
1152     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1153 }
1154
1155 my $kvm_api_version = 0;
1156
1157 sub kvm_version {
1158     return $kvm_api_version if $kvm_api_version;
1159
1160     open my $fh, '<', '/dev/kvm'
1161         or return;
1162
1163     # 0xae00 => KVM_GET_API_VERSION
1164     $kvm_api_version = ioctl($fh, 0xae00, 0);
1165
1166     return $kvm_api_version;
1167 }
1168
1169 my $kvm_user_version = {};
1170 my $kvm_mtime = {};
1171
1172 sub kvm_user_version {
1173     my ($binary) = @_;
1174
1175     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1176     my $st = stat($binary);
1177
1178     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1179     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1180         $cachedmtime == $st->mtime;
1181
1182     $kvm_user_version->{$binary} = 'unknown';
1183     $kvm_mtime->{$binary} = $st->mtime;
1184
1185     my $code = sub {
1186         my $line = shift;
1187         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1188             $kvm_user_version->{$binary} = $2;
1189         }
1190     };
1191
1192     eval { run_command([$binary, '--version'], outfunc => $code); };
1193     warn $@ if $@;
1194
1195     return $kvm_user_version->{$binary};
1196
1197 }
1198 my sub extract_version {
1199     my ($machine_type, $version) = @_;
1200     $version = kvm_user_version() if !defined($version);
1201     PVE::QemuServer::Machine::extract_version($machine_type, $version)
1202 }
1203
1204 sub kernel_has_vhost_net {
1205     return -c '/dev/vhost-net';
1206 }
1207
1208 sub option_exists {
1209     my $key = shift;
1210     return defined($confdesc->{$key});
1211 }
1212
1213 my $cdrom_path;
1214 sub get_cdrom_path {
1215
1216     return  $cdrom_path if $cdrom_path;
1217
1218     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1219     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1220     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1221 }
1222
1223 sub get_iso_path {
1224     my ($storecfg, $vmid, $cdrom) = @_;
1225
1226     if ($cdrom eq 'cdrom') {
1227         return get_cdrom_path();
1228     } elsif ($cdrom eq 'none') {
1229         return '';
1230     } elsif ($cdrom =~ m|^/|) {
1231         return $cdrom;
1232     } else {
1233         return PVE::Storage::path($storecfg, $cdrom);
1234     }
1235 }
1236
1237 # try to convert old style file names to volume IDs
1238 sub filename_to_volume_id {
1239     my ($vmid, $file, $media) = @_;
1240
1241      if (!($file eq 'none' || $file eq 'cdrom' ||
1242           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1243
1244         return if $file =~ m|/|;
1245
1246         if ($media && $media eq 'cdrom') {
1247             $file = "local:iso/$file";
1248         } else {
1249             $file = "local:$vmid/$file";
1250         }
1251     }
1252
1253     return $file;
1254 }
1255
1256 sub verify_media_type {
1257     my ($opt, $vtype, $media) = @_;
1258
1259     return if !$media;
1260
1261     my $etype;
1262     if ($media eq 'disk') {
1263         $etype = 'images';
1264     } elsif ($media eq 'cdrom') {
1265         $etype = 'iso';
1266     } else {
1267         die "internal error";
1268     }
1269
1270     return if ($vtype eq $etype);
1271
1272     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1273 }
1274
1275 sub cleanup_drive_path {
1276     my ($opt, $storecfg, $drive) = @_;
1277
1278     # try to convert filesystem paths to volume IDs
1279
1280     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1281         ($drive->{file} !~ m|^/dev/.+|) &&
1282         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1283         ($drive->{file} !~ m/^\d+$/)) {
1284         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1285         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1286             if !$vtype;
1287         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1288         verify_media_type($opt, $vtype, $drive->{media});
1289         $drive->{file} = $volid;
1290     }
1291
1292     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1293 }
1294
1295 sub parse_hotplug_features {
1296     my ($data) = @_;
1297
1298     my $res = {};
1299
1300     return $res if $data eq '0';
1301
1302     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1303
1304     foreach my $feature (PVE::Tools::split_list($data)) {
1305         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1306             $res->{$1} = 1;
1307         } else {
1308             die "invalid hotplug feature '$feature'\n";
1309         }
1310     }
1311     return $res;
1312 }
1313
1314 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1315 sub pve_verify_hotplug_features {
1316     my ($value, $noerr) = @_;
1317
1318     return $value if parse_hotplug_features($value);
1319
1320     return if $noerr;
1321
1322     die "unable to parse hotplug option\n";
1323 }
1324
1325 sub scsi_inquiry {
1326     my($fh, $noerr) = @_;
1327
1328     my $SG_IO = 0x2285;
1329     my $SG_GET_VERSION_NUM = 0x2282;
1330
1331     my $versionbuf = "\x00" x 8;
1332     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1333     if (!$ret) {
1334         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1335         return;
1336     }
1337     my $version = unpack("I", $versionbuf);
1338     if ($version < 30000) {
1339         die "scsi generic interface too old\n"  if !$noerr;
1340         return;
1341     }
1342
1343     my $buf = "\x00" x 36;
1344     my $sensebuf = "\x00" x 8;
1345     my $cmd = pack("C x3 C x1", 0x12, 36);
1346
1347     # see /usr/include/scsi/sg.h
1348     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1349
1350     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1351                       length($sensebuf), 0, length($buf), $buf,
1352                       $cmd, $sensebuf, 6000);
1353
1354     $ret = ioctl($fh, $SG_IO, $packet);
1355     if (!$ret) {
1356         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1357         return;
1358     }
1359
1360     my @res = unpack($sg_io_hdr_t, $packet);
1361     if ($res[17] || $res[18]) {
1362         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1363         return;
1364     }
1365
1366     my $res = {};
1367     (my $byte0, my $byte1, $res->{vendor},
1368      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1369
1370     $res->{removable} = $byte1 & 128 ? 1 : 0;
1371     $res->{type} = $byte0 & 31;
1372
1373     return $res;
1374 }
1375
1376 sub path_is_scsi {
1377     my ($path) = @_;
1378
1379     my $fh = IO::File->new("+<$path") || return;
1380     my $res = scsi_inquiry($fh, 1);
1381     close($fh);
1382
1383     return $res;
1384 }
1385
1386 sub print_tabletdevice_full {
1387     my ($conf, $arch) = @_;
1388
1389     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1390
1391     # we use uhci for old VMs because tablet driver was buggy in older qemu
1392     my $usbbus;
1393     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1394         $usbbus = 'ehci';
1395     } else {
1396         $usbbus = 'uhci';
1397     }
1398
1399     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1400 }
1401
1402 sub print_keyboarddevice_full {
1403     my ($conf, $arch, $machine) = @_;
1404
1405     return if $arch ne 'aarch64';
1406
1407     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1408 }
1409
1410 sub print_drivedevice_full {
1411     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1412
1413     my $device = '';
1414     my $maxdev = 0;
1415
1416     my $drive_id = "$drive->{interface}$drive->{index}";
1417     if ($drive->{interface} eq 'virtio') {
1418         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1419         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1420         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1421     } elsif ($drive->{interface} eq 'scsi') {
1422
1423         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1424         my $unit = $drive->{index} % $maxdev;
1425         my $devicetype = 'hd';
1426         my $path = '';
1427         if (drive_is_cdrom($drive)) {
1428             $devicetype = 'cd';
1429         } else {
1430             if ($drive->{file} =~ m|^/|) {
1431                 $path = $drive->{file};
1432                 if (my $info = path_is_scsi($path)) {
1433                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1434                         $devicetype = 'block';
1435                     } elsif ($info->{type} == 1) { # tape
1436                         $devicetype = 'generic';
1437                     }
1438                 }
1439             } else {
1440                  $path = PVE::Storage::path($storecfg, $drive->{file});
1441             }
1442
1443             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1444             my $version = extract_version($machine_type, kvm_user_version());
1445             if ($path =~ m/^iscsi\:\/\// &&
1446                !min_version($version, 4, 1)) {
1447                 $devicetype = 'generic';
1448             }
1449         }
1450
1451         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1452             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1453         } else {
1454             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1455                 .",lun=$drive->{index}";
1456         }
1457         $device .= ",drive=drive-$drive_id,id=$drive_id";
1458
1459         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1460             $device .= ",rotation_rate=1";
1461         }
1462         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1463
1464     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1465         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1466         my $controller = int($drive->{index} / $maxdev);
1467         my $unit = $drive->{index} % $maxdev;
1468         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1469
1470         $device = "ide-$devicetype";
1471         if ($drive->{interface} eq 'ide') {
1472             $device .= ",bus=ide.$controller,unit=$unit";
1473         } else {
1474             $device .= ",bus=ahci$controller.$unit";
1475         }
1476         $device .= ",drive=drive-$drive_id,id=$drive_id";
1477
1478         if ($devicetype eq 'hd') {
1479             if (my $model = $drive->{model}) {
1480                 $model = URI::Escape::uri_unescape($model);
1481                 $device .= ",model=$model";
1482             }
1483             if ($drive->{ssd}) {
1484                 $device .= ",rotation_rate=1";
1485             }
1486         }
1487         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1488     } elsif ($drive->{interface} eq 'usb') {
1489         die "implement me";
1490         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1491     } else {
1492         die "unsupported interface type";
1493     }
1494
1495     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1496
1497     if (my $serial = $drive->{serial}) {
1498         $serial = URI::Escape::uri_unescape($serial);
1499         $device .= ",serial=$serial";
1500     }
1501
1502
1503     return $device;
1504 }
1505
1506 sub get_initiator_name {
1507     my $initiator;
1508
1509     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1510     while (defined(my $line = <$fh>)) {
1511         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1512         $initiator = $1;
1513         last;
1514     }
1515     $fh->close();
1516
1517     return $initiator;
1518 }
1519
1520 sub print_drive_commandline_full {
1521     my ($storecfg, $vmid, $drive) = @_;
1522
1523     my $path;
1524     my $volid = $drive->{file};
1525     my $format;
1526
1527     if (drive_is_cdrom($drive)) {
1528         $path = get_iso_path($storecfg, $vmid, $volid);
1529     } else {
1530         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1531         if ($storeid) {
1532             $path = PVE::Storage::path($storecfg, $volid);
1533             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1534             $format = qemu_img_format($scfg, $volname);
1535         } else {
1536             $path = $volid;
1537             $format = "raw";
1538         }
1539    }
1540
1541     my $opts = '';
1542     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1543     foreach my $o (@qemu_drive_options) {
1544         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1545     }
1546
1547     # snapshot only accepts on|off
1548     if (defined($drive->{snapshot})) {
1549         my $v = $drive->{snapshot} ? 'on' : 'off';
1550         $opts .= ",snapshot=$v";
1551     }
1552
1553     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1554         my ($dir, $qmpname) = @$type;
1555         if (my $v = $drive->{"mbps$dir"}) {
1556             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1557         }
1558         if (my $v = $drive->{"mbps${dir}_max"}) {
1559             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1560         }
1561         if (my $v = $drive->{"bps${dir}_max_length"}) {
1562             $opts .= ",throttling.bps$qmpname-max-length=$v";
1563         }
1564         if (my $v = $drive->{"iops${dir}"}) {
1565             $opts .= ",throttling.iops$qmpname=$v";
1566         }
1567         if (my $v = $drive->{"iops${dir}_max"}) {
1568             $opts .= ",throttling.iops$qmpname-max=$v";
1569         }
1570         if (my $v = $drive->{"iops${dir}_max_length"}) {
1571             $opts .= ",throttling.iops$qmpname-max-length=$v";
1572         }
1573     }
1574
1575     $opts .= ",format=$format" if $format && !$drive->{format};
1576
1577     my $cache_direct = 0;
1578
1579     if (my $cache = $drive->{cache}) {
1580         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1581     } elsif (!drive_is_cdrom($drive)) {
1582         $opts .= ",cache=none";
1583         $cache_direct = 1;
1584     }
1585
1586     # aio native works only with O_DIRECT
1587     if (!$drive->{aio}) {
1588         if($cache_direct) {
1589             $opts .= ",aio=native";
1590         } else {
1591             $opts .= ",aio=threads";
1592         }
1593     }
1594
1595     if (!drive_is_cdrom($drive)) {
1596         my $detectzeroes;
1597         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1598             $detectzeroes = 'off';
1599         } elsif ($drive->{discard}) {
1600             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1601         } else {
1602             # This used to be our default with discard not being specified:
1603             $detectzeroes = 'on';
1604         }
1605         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1606     }
1607
1608     my $pathinfo = $path ? "file=$path," : '';
1609
1610     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1611 }
1612
1613 sub print_netdevice_full {
1614     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1615
1616     my $device = $net->{model};
1617     if ($net->{model} eq 'virtio') {
1618          $device = 'virtio-net-pci';
1619      };
1620
1621     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1622     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1623     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1624         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1625         # and out of each queue plus one config interrupt and control vector queue
1626         my $vectors = $net->{queues} * 2 + 2;
1627         $tmpstr .= ",vectors=$vectors,mq=on";
1628     }
1629     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1630
1631     if (my $mtu = $net->{mtu}) {
1632         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1633             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1634             if ($mtu == 1) {
1635                  $mtu = $bridge_mtu;
1636             } elsif ($mtu < 576) {
1637                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1638             } elsif ($mtu > $bridge_mtu) {
1639                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1640             }
1641             $tmpstr .= ",host_mtu=$mtu";
1642         } else {
1643             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1644         }
1645     }
1646
1647     if ($use_old_bios_files) {
1648         my $romfile;
1649         if ($device eq 'virtio-net-pci') {
1650             $romfile = 'pxe-virtio.rom';
1651         } elsif ($device eq 'e1000') {
1652             $romfile = 'pxe-e1000.rom';
1653         } elsif ($device eq 'ne2k') {
1654             $romfile = 'pxe-ne2k_pci.rom';
1655         } elsif ($device eq 'pcnet') {
1656             $romfile = 'pxe-pcnet.rom';
1657         } elsif ($device eq 'rtl8139') {
1658             $romfile = 'pxe-rtl8139.rom';
1659         }
1660         $tmpstr .= ",romfile=$romfile" if $romfile;
1661     }
1662
1663     return $tmpstr;
1664 }
1665
1666 sub print_netdev_full {
1667     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1668
1669     my $i = '';
1670     if ($netid =~ m/^net(\d+)$/) {
1671         $i = int($1);
1672     }
1673
1674     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1675
1676     my $ifname = "tap${vmid}i$i";
1677
1678     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1679     die "interface name '$ifname' is too long (max 15 character)\n"
1680         if length($ifname) >= 16;
1681
1682     my $vhostparam = '';
1683     if (is_native($arch)) {
1684         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1685     }
1686
1687     my $vmname = $conf->{name} || "vm$vmid";
1688
1689     my $netdev = "";
1690     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1691
1692     if ($net->{bridge}) {
1693         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1694             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1695     } else {
1696         $netdev = "type=user,id=$netid,hostname=$vmname";
1697     }
1698
1699     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1700
1701     return $netdev;
1702 }
1703
1704 my $vga_map = {
1705     'cirrus' => 'cirrus-vga',
1706     'std' => 'VGA',
1707     'vmware' => 'vmware-svga',
1708     'virtio' => 'virtio-vga',
1709 };
1710
1711 sub print_vga_device {
1712     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1713
1714     my $type = $vga_map->{$vga->{type}};
1715     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1716         $type = 'virtio-gpu';
1717     }
1718     my $vgamem_mb = $vga->{memory};
1719
1720     my $max_outputs = '';
1721     if ($qxlnum) {
1722         $type = $id ? 'qxl' : 'qxl-vga';
1723
1724         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1725             # set max outputs so linux can have up to 4 qxl displays with one device
1726             if (min_version($machine_version, 4, 1)) {
1727                 $max_outputs = ",max_outputs=4";
1728             }
1729         }
1730     }
1731
1732     die "no devicetype for $vga->{type}\n" if !$type;
1733
1734     my $memory = "";
1735     if ($vgamem_mb) {
1736         if ($vga->{type} eq 'virtio') {
1737             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1738             $memory = ",max_hostmem=$bytes";
1739         } elsif ($qxlnum) {
1740             # from https://www.spice-space.org/multiple-monitors.html
1741             $memory = ",vgamem_mb=$vga->{memory}";
1742             my $ram = $vgamem_mb * 4;
1743             my $vram = $vgamem_mb * 2;
1744             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1745         } else {
1746             $memory = ",vgamem_mb=$vga->{memory}";
1747         }
1748     } elsif ($qxlnum && $id) {
1749         $memory = ",ram_size=67108864,vram_size=33554432";
1750     }
1751
1752     my $edidoff = "";
1753     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1754         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1755     }
1756
1757     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1758     my $vgaid = "vga" . ($id // '');
1759     my $pciaddr;
1760
1761     if ($q35 && $vgaid eq 'vga') {
1762         # the first display uses pcie.0 bus on q35 machines
1763         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1764     } else {
1765         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1766     }
1767
1768     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1769 }
1770
1771 sub parse_number_sets {
1772     my ($set) = @_;
1773     my $res = [];
1774     foreach my $part (split(/;/, $set)) {
1775         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1776             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1777             push @$res, [ $1, $2 ];
1778         } else {
1779             die "invalid range: $part\n";
1780         }
1781     }
1782     return $res;
1783 }
1784
1785 sub parse_numa {
1786     my ($data) = @_;
1787
1788     my $res = parse_property_string($numa_fmt, $data);
1789     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1790     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1791     return $res;
1792 }
1793
1794 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1795 sub parse_net {
1796     my ($data) = @_;
1797
1798     my $res = eval { parse_property_string($net_fmt, $data) };
1799     if ($@) {
1800         warn $@;
1801         return;
1802     }
1803     if (!defined($res->{macaddr})) {
1804         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1805         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1806     }
1807     return $res;
1808 }
1809
1810 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1811 sub parse_ipconfig {
1812     my ($data) = @_;
1813
1814     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1815     if ($@) {
1816         warn $@;
1817         return;
1818     }
1819
1820     if ($res->{gw} && !$res->{ip}) {
1821         warn 'gateway specified without specifying an IP address';
1822         return;
1823     }
1824     if ($res->{gw6} && !$res->{ip6}) {
1825         warn 'IPv6 gateway specified without specifying an IPv6 address';
1826         return;
1827     }
1828     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1829         warn 'gateway specified together with DHCP';
1830         return;
1831     }
1832     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1833         # gw6 + auto/dhcp
1834         warn "IPv6 gateway specified together with $res->{ip6} address";
1835         return;
1836     }
1837
1838     if (!$res->{ip} && !$res->{ip6}) {
1839         return { ip => 'dhcp', ip6 => 'dhcp' };
1840     }
1841
1842     return $res;
1843 }
1844
1845 sub print_net {
1846     my $net = shift;
1847
1848     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1849 }
1850
1851 sub add_random_macs {
1852     my ($settings) = @_;
1853
1854     foreach my $opt (keys %$settings) {
1855         next if $opt !~ m/^net(\d+)$/;
1856         my $net = parse_net($settings->{$opt});
1857         next if !$net;
1858         $settings->{$opt} = print_net($net);
1859     }
1860 }
1861
1862 sub vm_is_volid_owner {
1863     my ($storecfg, $vmid, $volid) = @_;
1864
1865     if ($volid !~  m|^/|) {
1866         my ($path, $owner);
1867         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1868         if ($owner && ($owner == $vmid)) {
1869             return 1;
1870         }
1871     }
1872
1873     return;
1874 }
1875
1876 sub vmconfig_register_unused_drive {
1877     my ($storecfg, $vmid, $conf, $drive) = @_;
1878
1879     if (drive_is_cloudinit($drive)) {
1880         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1881         warn $@ if $@;
1882     } elsif (!drive_is_cdrom($drive)) {
1883         my $volid = $drive->{file};
1884         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1885             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1886         }
1887     }
1888 }
1889
1890 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1891 my $smbios1_fmt = {
1892     uuid => {
1893         type => 'string',
1894         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1895         format_description => 'UUID',
1896         description => "Set SMBIOS1 UUID.",
1897         optional => 1,
1898     },
1899     version => {
1900         type => 'string',
1901         pattern => '[A-Za-z0-9+\/]+={0,2}',
1902         format_description => 'Base64 encoded string',
1903         description => "Set SMBIOS1 version.",
1904         optional => 1,
1905     },
1906     serial => {
1907         type => 'string',
1908         pattern => '[A-Za-z0-9+\/]+={0,2}',
1909         format_description => 'Base64 encoded string',
1910         description => "Set SMBIOS1 serial number.",
1911         optional => 1,
1912     },
1913     manufacturer => {
1914         type => 'string',
1915         pattern => '[A-Za-z0-9+\/]+={0,2}',
1916         format_description => 'Base64 encoded string',
1917         description => "Set SMBIOS1 manufacturer.",
1918         optional => 1,
1919     },
1920     product => {
1921         type => 'string',
1922         pattern => '[A-Za-z0-9+\/]+={0,2}',
1923         format_description => 'Base64 encoded string',
1924         description => "Set SMBIOS1 product ID.",
1925         optional => 1,
1926     },
1927     sku => {
1928         type => 'string',
1929         pattern => '[A-Za-z0-9+\/]+={0,2}',
1930         format_description => 'Base64 encoded string',
1931         description => "Set SMBIOS1 SKU string.",
1932         optional => 1,
1933     },
1934     family => {
1935         type => 'string',
1936         pattern => '[A-Za-z0-9+\/]+={0,2}',
1937         format_description => 'Base64 encoded string',
1938         description => "Set SMBIOS1 family string.",
1939         optional => 1,
1940     },
1941     base64 => {
1942         type => 'boolean',
1943         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
1944         optional => 1,
1945     },
1946 };
1947
1948 sub parse_smbios1 {
1949     my ($data) = @_;
1950
1951     my $res = eval { parse_property_string($smbios1_fmt, $data) };
1952     warn $@ if $@;
1953     return $res;
1954 }
1955
1956 sub print_smbios1 {
1957     my ($smbios1) = @_;
1958     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1959 }
1960
1961 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1962
1963 sub parse_watchdog {
1964     my ($value) = @_;
1965
1966     return if !$value;
1967
1968     my $res = eval { parse_property_string($watchdog_fmt, $value) };
1969     warn $@ if $@;
1970     return $res;
1971 }
1972
1973 sub parse_guest_agent {
1974     my ($value) = @_;
1975
1976     return {} if !defined($value->{agent});
1977
1978     my $res = eval { parse_property_string($agent_fmt, $value->{agent}) };
1979     warn $@ if $@;
1980
1981     # if the agent is disabled ignore the other potentially set properties
1982     return {} if !$res->{enabled};
1983     return $res;
1984 }
1985
1986 sub parse_vga {
1987     my ($value) = @_;
1988
1989     return {} if !$value;
1990     my $res = eval { parse_property_string($vga_fmt, $value) };
1991     warn $@ if $@;
1992     return $res;
1993 }
1994
1995 sub parse_rng {
1996     my ($value) = @_;
1997
1998     return if !$value;
1999
2000     my $res = eval { parse_property_string($rng_fmt, $value) };
2001     warn $@ if $@;
2002     return $res;
2003 }
2004
2005 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2006 sub verify_usb_device {
2007     my ($value, $noerr) = @_;
2008
2009     return $value if parse_usb_device($value);
2010
2011     return if $noerr;
2012
2013     die "unable to parse usb device\n";
2014 }
2015
2016 # add JSON properties for create and set function
2017 sub json_config_properties {
2018     my $prop = shift;
2019
2020     foreach my $opt (keys %$confdesc) {
2021         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' ||
2022             $opt eq 'runningmachine' || $opt eq 'runningcpu';
2023         $prop->{$opt} = $confdesc->{$opt};
2024     }
2025
2026     return $prop;
2027 }
2028
2029 # return copy of $confdesc_cloudinit to generate documentation
2030 sub cloudinit_config_properties {
2031
2032     return dclone($confdesc_cloudinit);
2033 }
2034
2035 sub check_type {
2036     my ($key, $value) = @_;
2037
2038     die "unknown setting '$key'\n" if !$confdesc->{$key};
2039
2040     my $type = $confdesc->{$key}->{type};
2041
2042     if (!defined($value)) {
2043         die "got undefined value\n";
2044     }
2045
2046     if ($value =~ m/[\n\r]/) {
2047         die "property contains a line feed\n";
2048     }
2049
2050     if ($type eq 'boolean') {
2051         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2052         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2053         die "type check ('boolean') failed - got '$value'\n";
2054     } elsif ($type eq 'integer') {
2055         return int($1) if $value =~ m/^(\d+)$/;
2056         die "type check ('integer') failed - got '$value'\n";
2057     } elsif ($type eq 'number') {
2058         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2059         die "type check ('number') failed - got '$value'\n";
2060     } elsif ($type eq 'string') {
2061         if (my $fmt = $confdesc->{$key}->{format}) {
2062             PVE::JSONSchema::check_format($fmt, $value);
2063             return $value;
2064         }
2065         $value =~ s/^\"(.*)\"$/$1/;
2066         return $value;
2067     } else {
2068         die "internal error"
2069     }
2070 }
2071
2072 sub destroy_vm {
2073     my ($storecfg, $vmid, $skiplock, $replacement_conf) = @_;
2074
2075     my $conf = PVE::QemuConfig->load_config($vmid);
2076
2077     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2078
2079     if ($conf->{template}) {
2080         # check if any base image is still used by a linked clone
2081         PVE::QemuConfig->foreach_volume($conf, sub {
2082                 my ($ds, $drive) = @_;
2083                 return if drive_is_cdrom($drive);
2084
2085                 my $volid = $drive->{file};
2086                 return if !$volid || $volid =~ m|^/|;
2087
2088                 die "base volume '$volid' is still in use by linked cloned\n"
2089                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2090
2091         });
2092     }
2093
2094     # only remove disks owned by this VM
2095     PVE::QemuConfig->foreach_volume($conf, sub {
2096         my ($ds, $drive) = @_;
2097         return if drive_is_cdrom($drive, 1);
2098
2099         my $volid = $drive->{file};
2100         return if !$volid || $volid =~ m|^/|;
2101
2102         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2103         return if !$path || !$owner || ($owner != $vmid);
2104
2105         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2106         warn "Could not remove disk '$volid', check manually: $@" if $@;
2107     });
2108
2109     # also remove unused disk
2110     my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2111     PVE::Storage::foreach_volid($vmdisks, sub {
2112         my ($volid, $sid, $volname, $d) = @_;
2113         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2114         warn $@ if $@;
2115     });
2116
2117     if (defined $replacement_conf) {
2118         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2119     } else {
2120         PVE::QemuConfig->destroy_config($vmid);
2121     }
2122 }
2123
2124 sub parse_vm_config {
2125     my ($filename, $raw) = @_;
2126
2127     return if !defined($raw);
2128
2129     my $res = {
2130         digest => Digest::SHA::sha1_hex($raw),
2131         snapshots => {},
2132         pending => {},
2133     };
2134
2135     $filename =~ m|/qemu-server/(\d+)\.conf$|
2136         || die "got strange filename '$filename'";
2137
2138     my $vmid = $1;
2139
2140     my $conf = $res;
2141     my $descr;
2142     my $section = '';
2143
2144     my @lines = split(/\n/, $raw);
2145     foreach my $line (@lines) {
2146         next if $line =~ m/^\s*$/;
2147
2148         if ($line =~ m/^\[PENDING\]\s*$/i) {
2149             $section = 'pending';
2150             if (defined($descr)) {
2151                 $descr =~ s/\s+$//;
2152                 $conf->{description} = $descr;
2153             }
2154             $descr = undef;
2155             $conf = $res->{$section} = {};
2156             next;
2157
2158         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2159             $section = $1;
2160             if (defined($descr)) {
2161                 $descr =~ s/\s+$//;
2162                 $conf->{description} = $descr;
2163             }
2164             $descr = undef;
2165             $conf = $res->{snapshots}->{$section} = {};
2166             next;
2167         }
2168
2169         if ($line =~ m/^\#(.*)\s*$/) {
2170             $descr = '' if !defined($descr);
2171             $descr .= PVE::Tools::decode_text($1) . "\n";
2172             next;
2173         }
2174
2175         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2176             $descr = '' if !defined($descr);
2177             $descr .= PVE::Tools::decode_text($2);
2178         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2179             $conf->{snapstate} = $1;
2180         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2181             my $key = $1;
2182             my $value = $2;
2183             $conf->{$key} = $value;
2184         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2185             my $value = $1;
2186             if ($section eq 'pending') {
2187                 $conf->{delete} = $value; # we parse this later
2188             } else {
2189                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2190             }
2191         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2192             my $key = $1;
2193             my $value = $2;
2194             eval { $value = check_type($key, $value); };
2195             if ($@) {
2196                 warn "vm $vmid - unable to parse value of '$key' - $@";
2197             } else {
2198                 $key = 'ide2' if $key eq 'cdrom';
2199                 my $fmt = $confdesc->{$key}->{format};
2200                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2201                     my $v = parse_drive($key, $value);
2202                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2203                         $v->{file} = $volid;
2204                         $value = print_drive($v);
2205                     } else {
2206                         warn "vm $vmid - unable to parse value of '$key'\n";
2207                         next;
2208                     }
2209                 }
2210
2211                 $conf->{$key} = $value;
2212             }
2213         }
2214     }
2215
2216     if (defined($descr)) {
2217         $descr =~ s/\s+$//;
2218         $conf->{description} = $descr;
2219     }
2220     delete $res->{snapstate}; # just to be sure
2221
2222     return $res;
2223 }
2224
2225 sub write_vm_config {
2226     my ($filename, $conf) = @_;
2227
2228     delete $conf->{snapstate}; # just to be sure
2229
2230     if ($conf->{cdrom}) {
2231         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2232         $conf->{ide2} = $conf->{cdrom};
2233         delete $conf->{cdrom};
2234     }
2235
2236     # we do not use 'smp' any longer
2237     if ($conf->{sockets}) {
2238         delete $conf->{smp};
2239     } elsif ($conf->{smp}) {
2240         $conf->{sockets} = $conf->{smp};
2241         delete $conf->{cores};
2242         delete $conf->{smp};
2243     }
2244
2245     my $used_volids = {};
2246
2247     my $cleanup_config = sub {
2248         my ($cref, $pending, $snapname) = @_;
2249
2250         foreach my $key (keys %$cref) {
2251             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2252                 $key eq 'snapstate' || $key eq 'pending';
2253             my $value = $cref->{$key};
2254             if ($key eq 'delete') {
2255                 die "propertry 'delete' is only allowed in [PENDING]\n"
2256                     if !$pending;
2257                 # fixme: check syntax?
2258                 next;
2259             }
2260             eval { $value = check_type($key, $value); };
2261             die "unable to parse value of '$key' - $@" if $@;
2262
2263             $cref->{$key} = $value;
2264
2265             if (!$snapname && is_valid_drivename($key)) {
2266                 my $drive = parse_drive($key, $value);
2267                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2268             }
2269         }
2270     };
2271
2272     &$cleanup_config($conf);
2273
2274     &$cleanup_config($conf->{pending}, 1);
2275
2276     foreach my $snapname (keys %{$conf->{snapshots}}) {
2277         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2278         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2279     }
2280
2281     # remove 'unusedX' settings if we re-add a volume
2282     foreach my $key (keys %$conf) {
2283         my $value = $conf->{$key};
2284         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2285             delete $conf->{$key};
2286         }
2287     }
2288
2289     my $generate_raw_config = sub {
2290         my ($conf, $pending) = @_;
2291
2292         my $raw = '';
2293
2294         # add description as comment to top of file
2295         if (defined(my $descr = $conf->{description})) {
2296             if ($descr) {
2297                 foreach my $cl (split(/\n/, $descr)) {
2298                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2299                 }
2300             } else {
2301                 $raw .= "#\n" if $pending;
2302             }
2303         }
2304
2305         foreach my $key (sort keys %$conf) {
2306             next if $key =~ /^(digest|description|pending|snapshots)$/;
2307             $raw .= "$key: $conf->{$key}\n";
2308         }
2309         return $raw;
2310     };
2311
2312     my $raw = &$generate_raw_config($conf);
2313
2314     if (scalar(keys %{$conf->{pending}})){
2315         $raw .= "\n[PENDING]\n";
2316         $raw .= &$generate_raw_config($conf->{pending}, 1);
2317     }
2318
2319     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2320         $raw .= "\n[$snapname]\n";
2321         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2322     }
2323
2324     return $raw;
2325 }
2326
2327 sub load_defaults {
2328
2329     my $res = {};
2330
2331     # we use static defaults from our JSON schema configuration
2332     foreach my $key (keys %$confdesc) {
2333         if (defined(my $default = $confdesc->{$key}->{default})) {
2334             $res->{$key} = $default;
2335         }
2336     }
2337
2338     return $res;
2339 }
2340
2341 sub config_list {
2342     my $vmlist = PVE::Cluster::get_vmlist();
2343     my $res = {};
2344     return $res if !$vmlist || !$vmlist->{ids};
2345     my $ids = $vmlist->{ids};
2346     my $nodename = nodename();
2347
2348     foreach my $vmid (keys %$ids) {
2349         my $d = $ids->{$vmid};
2350         next if !$d->{node} || $d->{node} ne $nodename;
2351         next if !$d->{type} || $d->{type} ne 'qemu';
2352         $res->{$vmid}->{exists} = 1;
2353     }
2354     return $res;
2355 }
2356
2357 # test if VM uses local resources (to prevent migration)
2358 sub check_local_resources {
2359     my ($conf, $noerr) = @_;
2360
2361     my @loc_res = ();
2362
2363     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2364     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2365
2366     push @loc_res, "ivshmem" if $conf->{ivshmem};
2367
2368     foreach my $k (keys %$conf) {
2369         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2370         # sockets are safe: they will recreated be on the target side post-migrate
2371         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2372         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2373     }
2374
2375     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2376
2377     return \@loc_res;
2378 }
2379
2380 # check if used storages are available on all nodes (use by migrate)
2381 sub check_storage_availability {
2382     my ($storecfg, $conf, $node) = @_;
2383
2384     PVE::QemuConfig->foreach_volume($conf, sub {
2385         my ($ds, $drive) = @_;
2386
2387         my $volid = $drive->{file};
2388         return if !$volid;
2389
2390         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2391         return if !$sid;
2392
2393         # check if storage is available on both nodes
2394         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2395         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2396    });
2397 }
2398
2399 # list nodes where all VM images are available (used by has_feature API)
2400 sub shared_nodes {
2401     my ($conf, $storecfg) = @_;
2402
2403     my $nodelist = PVE::Cluster::get_nodelist();
2404     my $nodehash = { map { $_ => 1 } @$nodelist };
2405     my $nodename = nodename();
2406
2407     PVE::QemuConfig->foreach_volume($conf, sub {
2408         my ($ds, $drive) = @_;
2409
2410         my $volid = $drive->{file};
2411         return if !$volid;
2412
2413         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2414         if ($storeid) {
2415             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2416             if ($scfg->{disable}) {
2417                 $nodehash = {};
2418             } elsif (my $avail = $scfg->{nodes}) {
2419                 foreach my $node (keys %$nodehash) {
2420                     delete $nodehash->{$node} if !$avail->{$node};
2421                 }
2422             } elsif (!$scfg->{shared}) {
2423                 foreach my $node (keys %$nodehash) {
2424                     delete $nodehash->{$node} if $node ne $nodename
2425                 }
2426             }
2427         }
2428     });
2429
2430     return $nodehash
2431 }
2432
2433 sub check_local_storage_availability {
2434     my ($conf, $storecfg) = @_;
2435
2436     my $nodelist = PVE::Cluster::get_nodelist();
2437     my $nodehash = { map { $_ => {} } @$nodelist };
2438
2439     PVE::QemuConfig->foreach_volume($conf, sub {
2440         my ($ds, $drive) = @_;
2441
2442         my $volid = $drive->{file};
2443         return if !$volid;
2444
2445         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2446         if ($storeid) {
2447             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2448
2449             if ($scfg->{disable}) {
2450                 foreach my $node (keys %$nodehash) {
2451                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2452                 }
2453             } elsif (my $avail = $scfg->{nodes}) {
2454                 foreach my $node (keys %$nodehash) {
2455                     if (!$avail->{$node}) {
2456                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2457                     }
2458                 }
2459             }
2460         }
2461     });
2462
2463     foreach my $node (values %$nodehash) {
2464         if (my $unavail = $node->{unavailable_storages}) {
2465             $node->{unavailable_storages} = [ sort keys %$unavail ];
2466         }
2467     }
2468
2469     return $nodehash
2470 }
2471
2472 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2473 sub check_running {
2474     my ($vmid, $nocheck, $node) = @_;
2475
2476     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2477     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2478 }
2479
2480 sub vzlist {
2481
2482     my $vzlist = config_list();
2483
2484     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2485
2486     while (defined(my $de = $fd->read)) {
2487         next if $de !~ m/^(\d+)\.pid$/;
2488         my $vmid = $1;
2489         next if !defined($vzlist->{$vmid});
2490         if (my $pid = check_running($vmid)) {
2491             $vzlist->{$vmid}->{pid} = $pid;
2492         }
2493     }
2494
2495     return $vzlist;
2496 }
2497
2498 our $vmstatus_return_properties = {
2499     vmid => get_standard_option('pve-vmid'),
2500     status => {
2501         description => "Qemu process status.",
2502         type => 'string',
2503         enum => ['stopped', 'running'],
2504     },
2505     maxmem => {
2506         description => "Maximum memory in bytes.",
2507         type => 'integer',
2508         optional => 1,
2509         renderer => 'bytes',
2510     },
2511     maxdisk => {
2512         description => "Root disk size in bytes.",
2513         type => 'integer',
2514         optional => 1,
2515         renderer => 'bytes',
2516     },
2517     name => {
2518         description => "VM name.",
2519         type => 'string',
2520         optional => 1,
2521     },
2522     qmpstatus => {
2523         description => "Qemu QMP agent status.",
2524         type => 'string',
2525         optional => 1,
2526     },
2527     pid => {
2528         description => "PID of running qemu process.",
2529         type => 'integer',
2530         optional => 1,
2531     },
2532     uptime => {
2533         description => "Uptime.",
2534         type => 'integer',
2535         optional => 1,
2536         renderer => 'duration',
2537     },
2538     cpus => {
2539         description => "Maximum usable CPUs.",
2540         type => 'number',
2541         optional => 1,
2542     },
2543     lock => {
2544         description => "The current config lock, if any.",
2545         type => 'string',
2546         optional => 1,
2547     },
2548     tags => {
2549         description  => "The current configured tags, if any",
2550         type => 'string',
2551         optional => 1,
2552     },
2553 };
2554
2555 my $last_proc_pid_stat;
2556
2557 # get VM status information
2558 # This must be fast and should not block ($full == false)
2559 # We only query KVM using QMP if $full == true (this can be slow)
2560 sub vmstatus {
2561     my ($opt_vmid, $full) = @_;
2562
2563     my $res = {};
2564
2565     my $storecfg = PVE::Storage::config();
2566
2567     my $list = vzlist();
2568     my $defaults = load_defaults();
2569
2570     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2571
2572     my $cpucount = $cpuinfo->{cpus} || 1;
2573
2574     foreach my $vmid (keys %$list) {
2575         next if $opt_vmid && ($vmid ne $opt_vmid);
2576
2577         my $conf = PVE::QemuConfig->load_config($vmid);
2578
2579         my $d = { vmid => $vmid };
2580         $d->{pid} = $list->{$vmid}->{pid};
2581
2582         # fixme: better status?
2583         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2584
2585         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2586         if (defined($size)) {
2587             $d->{disk} = 0; # no info available
2588             $d->{maxdisk} = $size;
2589         } else {
2590             $d->{disk} = 0;
2591             $d->{maxdisk} = 0;
2592         }
2593
2594         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2595             * ($conf->{cores} || $defaults->{cores});
2596         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2597         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2598
2599         $d->{name} = $conf->{name} || "VM $vmid";
2600         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2601             : $defaults->{memory}*(1024*1024);
2602
2603         if ($conf->{balloon}) {
2604             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2605             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2606                 : $defaults->{shares};
2607         }
2608
2609         $d->{uptime} = 0;
2610         $d->{cpu} = 0;
2611         $d->{mem} = 0;
2612
2613         $d->{netout} = 0;
2614         $d->{netin} = 0;
2615
2616         $d->{diskread} = 0;
2617         $d->{diskwrite} = 0;
2618
2619         $d->{template} = PVE::QemuConfig->is_template($conf);
2620
2621         $d->{serial} = 1 if conf_has_serial($conf);
2622         $d->{lock} = $conf->{lock} if $conf->{lock};
2623         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2624
2625         $res->{$vmid} = $d;
2626     }
2627
2628     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2629     foreach my $dev (keys %$netdev) {
2630         next if $dev !~ m/^tap([1-9]\d*)i/;
2631         my $vmid = $1;
2632         my $d = $res->{$vmid};
2633         next if !$d;
2634
2635         $d->{netout} += $netdev->{$dev}->{receive};
2636         $d->{netin} += $netdev->{$dev}->{transmit};
2637
2638         if ($full) {
2639             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2640             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2641         }
2642
2643     }
2644
2645     my $ctime = gettimeofday;
2646
2647     foreach my $vmid (keys %$list) {
2648
2649         my $d = $res->{$vmid};
2650         my $pid = $d->{pid};
2651         next if !$pid;
2652
2653         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2654         next if !$pstat; # not running
2655
2656         my $used = $pstat->{utime} + $pstat->{stime};
2657
2658         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2659
2660         if ($pstat->{vsize}) {
2661             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2662         }
2663
2664         my $old = $last_proc_pid_stat->{$pid};
2665         if (!$old) {
2666             $last_proc_pid_stat->{$pid} = {
2667                 time => $ctime,
2668                 used => $used,
2669                 cpu => 0,
2670             };
2671             next;
2672         }
2673
2674         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2675
2676         if ($dtime > 1000) {
2677             my $dutime = $used -  $old->{used};
2678
2679             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2680             $last_proc_pid_stat->{$pid} = {
2681                 time => $ctime,
2682                 used => $used,
2683                 cpu => $d->{cpu},
2684             };
2685         } else {
2686             $d->{cpu} = $old->{cpu};
2687         }
2688     }
2689
2690     return $res if !$full;
2691
2692     my $qmpclient = PVE::QMPClient->new();
2693
2694     my $ballooncb = sub {
2695         my ($vmid, $resp) = @_;
2696
2697         my $info = $resp->{'return'};
2698         return if !$info->{max_mem};
2699
2700         my $d = $res->{$vmid};
2701
2702         # use memory assigned to VM
2703         $d->{maxmem} = $info->{max_mem};
2704         $d->{balloon} = $info->{actual};
2705
2706         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2707             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2708             $d->{freemem} = $info->{free_mem};
2709         }
2710
2711         $d->{ballooninfo} = $info;
2712     };
2713
2714     my $blockstatscb = sub {
2715         my ($vmid, $resp) = @_;
2716         my $data = $resp->{'return'} || [];
2717         my $totalrdbytes = 0;
2718         my $totalwrbytes = 0;
2719
2720         for my $blockstat (@$data) {
2721             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2722             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2723
2724             $blockstat->{device} =~ s/drive-//;
2725             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2726         }
2727         $res->{$vmid}->{diskread} = $totalrdbytes;
2728         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2729     };
2730
2731     my $statuscb = sub {
2732         my ($vmid, $resp) = @_;
2733
2734         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2735         # this fails if ballon driver is not loaded, so this must be
2736         # the last commnand (following command are aborted if this fails).
2737         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2738
2739         my $status = 'unknown';
2740         if (!defined($status = $resp->{'return'}->{status})) {
2741             warn "unable to get VM status\n";
2742             return;
2743         }
2744
2745         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2746     };
2747
2748     foreach my $vmid (keys %$list) {
2749         next if $opt_vmid && ($vmid ne $opt_vmid);
2750         next if !$res->{$vmid}->{pid}; # not running
2751         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2752     }
2753
2754     $qmpclient->queue_execute(undef, 2);
2755
2756     foreach my $vmid (keys %$list) {
2757         next if $opt_vmid && ($vmid ne $opt_vmid);
2758         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2759     }
2760
2761     return $res;
2762 }
2763
2764 sub conf_has_serial {
2765     my ($conf) = @_;
2766
2767     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2768         if ($conf->{"serial$i"}) {
2769             return 1;
2770         }
2771     }
2772
2773     return 0;
2774 }
2775
2776 sub conf_has_audio {
2777     my ($conf, $id) = @_;
2778
2779     $id //= 0;
2780     my $audio = $conf->{"audio$id"};
2781     return if !defined($audio);
2782
2783     my $audioproperties = parse_property_string($audio_fmt, $audio);
2784     my $audiodriver = $audioproperties->{driver} // 'spice';
2785
2786     return {
2787         dev => $audioproperties->{device},
2788         dev_id => "audiodev$id",
2789         backend => $audiodriver,
2790         backend_id => "$audiodriver-backend${id}",
2791     };
2792 }
2793
2794 sub audio_devs {
2795     my ($audio, $audiopciaddr, $machine_version) = @_;
2796
2797     my $devs = [];
2798
2799     my $id = $audio->{dev_id};
2800     my $audiodev = "";
2801     if (min_version($machine_version, 4, 2)) {
2802         $audiodev = ",audiodev=$audio->{backend_id}";
2803     }
2804
2805     if ($audio->{dev} eq 'AC97') {
2806         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
2807     } elsif ($audio->{dev} =~ /intel\-hda$/) {
2808         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
2809         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
2810         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
2811     } else {
2812         die "unkown audio device '$audio->{dev}', implement me!";
2813     }
2814
2815     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
2816
2817     return $devs;
2818 }
2819
2820 sub vga_conf_has_spice {
2821     my ($vga) = @_;
2822
2823     my $vgaconf = parse_vga($vga);
2824     my $vgatype = $vgaconf->{type};
2825     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
2826
2827     return $1 || 1;
2828 }
2829
2830 sub is_native($) {
2831     my ($arch) = @_;
2832     return get_host_arch() eq $arch;
2833 }
2834
2835 sub get_vm_arch {
2836     my ($conf) = @_;
2837     return $conf->{arch} // get_host_arch();
2838 }
2839
2840 my $default_machines = {
2841     x86_64 => 'pc',
2842     aarch64 => 'virt',
2843 };
2844
2845 sub get_vm_machine {
2846     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
2847
2848     my $machine = $forcemachine || $conf->{machine};
2849
2850     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
2851         $arch //= 'x86_64';
2852         $machine ||= $default_machines->{$arch};
2853         if ($add_pve_version) {
2854             $kvmversion //= kvm_user_version();
2855             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
2856             $machine .= "+pve$pvever";
2857         }
2858     }
2859
2860     if ($add_pve_version && $machine !~ m/\+pve\d+$/) {
2861         # for version-pinned machines that do not include a pve-version (e.g.
2862         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
2863         $machine .= '+pve0';
2864     }
2865
2866     return $machine;
2867 }
2868
2869 sub get_ovmf_files($) {
2870     my ($arch) = @_;
2871
2872     my $ovmf = $OVMF->{$arch}
2873         or die "no OVMF images known for architecture '$arch'\n";
2874
2875     return @$ovmf;
2876 }
2877
2878 my $Arch2Qemu = {
2879     aarch64 => '/usr/bin/qemu-system-aarch64',
2880     x86_64 => '/usr/bin/qemu-system-x86_64',
2881 };
2882 sub get_command_for_arch($) {
2883     my ($arch) = @_;
2884     return '/usr/bin/kvm' if is_native($arch);
2885
2886     my $cmd = $Arch2Qemu->{$arch}
2887         or die "don't know how to emulate architecture '$arch'\n";
2888     return $cmd;
2889 }
2890
2891 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
2892 # to use in a QEMU command line (-cpu element), first array_intersect the result
2893 # of query_supported_ with query_understood_. This is necessary because:
2894 #
2895 # a) query_understood_ returns flags the host cannot use and
2896 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
2897 #    flags, but CPU settings - with most of them being flags. Those settings
2898 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
2899 #
2900 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
2901 # expensive. If you need the value returned from this, you can get it much
2902 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
2903 # $accel being 'kvm' or 'tcg'.
2904 #
2905 # pvestatd calls this function on startup and whenever the QEMU/KVM version
2906 # changes, automatically populating pmxcfs.
2907 #
2908 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
2909 # since kvm and tcg machines support different flags
2910 #
2911 sub query_supported_cpu_flags {
2912     my ($arch) = @_;
2913
2914     $arch //= get_host_arch();
2915     my $default_machine = $default_machines->{$arch};
2916
2917     my $flags = {};
2918
2919     # FIXME: Once this is merged, the code below should work for ARM as well:
2920     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
2921     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
2922         $arch eq "aarch64";
2923
2924     my $kvm_supported = defined(kvm_version());
2925     my $qemu_cmd = get_command_for_arch($arch);
2926     my $fakevmid = -1;
2927     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
2928
2929     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
2930     my $query_supported_run_qemu = sub {
2931         my ($kvm) = @_;
2932
2933         my $flags = {};
2934         my $cmd = [
2935             $qemu_cmd,
2936             '-machine', $default_machine,
2937             '-display', 'none',
2938             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server,nowait",
2939             '-mon', 'chardev=qmp,mode=control',
2940             '-pidfile', $pidfile,
2941             '-S', '-daemonize'
2942         ];
2943
2944         if (!$kvm) {
2945             push @$cmd, '-accel', 'tcg';
2946         }
2947
2948         my $rc = run_command($cmd, noerr => 1, quiet => 0);
2949         die "QEMU flag querying VM exited with code " . $rc if $rc;
2950
2951         eval {
2952             my $cmd_result = mon_cmd(
2953                 $fakevmid,
2954                 'query-cpu-model-expansion',
2955                 type => 'full',
2956                 model => { name => 'host' }
2957             );
2958
2959             my $props = $cmd_result->{model}->{props};
2960             foreach my $prop (keys %$props) {
2961                 next if $props->{$prop} ne '1';
2962                 # QEMU returns some flags multiple times, with '_', '.' or '-'
2963                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
2964                 # We only keep those with underscores, to match /proc/cpuinfo
2965                 $prop =~ s/\.|-/_/g;
2966                 $flags->{$prop} = 1;
2967             }
2968         };
2969         my $err = $@;
2970
2971         # force stop with 10 sec timeout and 'nocheck'
2972         # always stop, even if QMP failed
2973         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
2974
2975         die $err if $err;
2976
2977         return [ sort keys %$flags ];
2978     };
2979
2980     # We need to query QEMU twice, since KVM and TCG have different supported flags
2981     PVE::QemuConfig->lock_config($fakevmid, sub {
2982         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
2983         warn "warning: failed querying supported tcg flags: $@\n" if $@;
2984
2985         if ($kvm_supported) {
2986             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
2987             warn "warning: failed querying supported kvm flags: $@\n" if $@;
2988         }
2989     });
2990
2991     return $flags;
2992 }
2993
2994 # Understood CPU flags are written to a file at 'pve-qemu' compile time
2995 my $understood_cpu_flag_dir = "/usr/share/kvm";
2996 sub query_understood_cpu_flags {
2997     my $arch = get_host_arch();
2998     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
2999
3000     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3001         if ! -e $filepath;
3002
3003     my $raw = file_get_contents($filepath);
3004     $raw =~ s/^\s+|\s+$//g;
3005     my @flags = split(/\s+/, $raw);
3006
3007     return \@flags;
3008 }
3009
3010 sub config_to_command {
3011     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu) = @_;
3012
3013     my $cmd = [];
3014     my $globalFlags = [];
3015     my $machineFlags = [];
3016     my $rtcFlags = [];
3017     my $devices = [];
3018     my $pciaddr = '';
3019     my $bridges = {};
3020     my $ostype = $conf->{ostype};
3021     my $winversion = windows_version($ostype);
3022     my $kvm = $conf->{kvm};
3023     my $nodename = nodename();
3024
3025     my $arch = get_vm_arch($conf);
3026     my $kvm_binary = get_command_for_arch($arch);
3027     my $kvmver = kvm_user_version($kvm_binary);
3028
3029     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3030         $kvmver //= "undefined";
3031         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3032     }
3033
3034     my $add_pve_version = min_version($kvmver, 4, 1);
3035
3036     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3037     my $machine_version = extract_version($machine_type, $kvmver);
3038     $kvm //= 1 if is_native($arch);
3039
3040     $machine_version =~ m/(\d+)\.(\d+)/;
3041     my ($machine_major, $machine_minor) = ($1, $2);
3042
3043     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3044         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3045     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3046         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3047             ." please upgrade node '$nodename'\n"
3048     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3049         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3050         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3051             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3052             ." node '$nodename'\n";
3053     }
3054
3055     # if a specific +pve version is required for a feature, use $version_guard
3056     # instead of min_version to allow machines to be run with the minimum
3057     # required version
3058     my $required_pve_version = 0;
3059     my $version_guard = sub {
3060         my ($major, $minor, $pve) = @_;
3061         return 0 if !min_version($machine_version, $major, $minor, $pve);
3062         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3063         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3064         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3065         return 1;
3066     };
3067
3068     if ($kvm && !defined kvm_version()) {
3069         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3070             ." or enable in BIOS.\n";
3071     }
3072
3073     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3074     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3075     my $use_old_bios_files = undef;
3076     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3077
3078     my $cpuunits = defined($conf->{cpuunits}) ?
3079             $conf->{cpuunits} : $defaults->{cpuunits};
3080
3081     push @$cmd, $kvm_binary;
3082
3083     push @$cmd, '-id', $vmid;
3084
3085     my $vmname = $conf->{name} || "vm$vmid";
3086
3087     push @$cmd, '-name', $vmname;
3088
3089     my $use_virtio = 0;
3090
3091     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3092     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3093     push @$cmd, '-mon', "chardev=qmp,mode=control";
3094
3095     if (min_version($machine_version, 2, 12)) {
3096         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3097         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3098     }
3099
3100     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3101
3102     push @$cmd, '-daemonize';
3103
3104     if ($conf->{smbios1}) {
3105         my $smbios_conf = parse_smbios1($conf->{smbios1});
3106         if ($smbios_conf->{base64}) {
3107             # Do not pass base64 flag to qemu
3108             delete $smbios_conf->{base64};
3109             my $smbios_string = "";
3110             foreach my $key (keys %$smbios_conf) {
3111                 my $value;
3112                 if ($key eq "uuid") {
3113                     $value = $smbios_conf->{uuid}
3114                 } else {
3115                     $value = decode_base64($smbios_conf->{$key});
3116                 }
3117                 # qemu accepts any binary data, only commas need escaping by double comma
3118                 $value =~ s/,/,,/g;
3119                 $smbios_string .= "," . $key . "=" . $value if $value;
3120             }
3121             push @$cmd, '-smbios', "type=1" . $smbios_string;
3122         } else {
3123             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3124         }
3125     }
3126
3127     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3128         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3129         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3130
3131         my ($path, $format);
3132         if (my $efidisk = $conf->{efidisk0}) {
3133             my $d = parse_drive('efidisk0', $efidisk);
3134             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3135             $format = $d->{format};
3136             if ($storeid) {
3137                 $path = PVE::Storage::path($storecfg, $d->{file});
3138                 if (!defined($format)) {
3139                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3140                     $format = qemu_img_format($scfg, $volname);
3141                 }
3142             } else {
3143                 $path = $d->{file};
3144                 die "efidisk format must be specified\n"
3145                     if !defined($format);
3146             }
3147         } else {
3148             warn "no efidisk configured! Using temporary efivars disk.\n";
3149             $path = "/tmp/$vmid-ovmf.fd";
3150             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3151             $format = 'raw';
3152         }
3153
3154         my $size_str = "";
3155
3156         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3157             $size_str = ",size=" . (-s $ovmf_vars);
3158         }
3159
3160         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3161         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0$size_str,file=$path";
3162     }
3163
3164     # load q35 config
3165     if ($q35) {
3166         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3167         if (min_version($machine_version, 4, 0)) {
3168             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3169         } else {
3170             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3171         }
3172     }
3173
3174     if ($conf->{vmgenid}) {
3175         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3176     }
3177
3178     # add usb controllers
3179     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3180         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3181     push @$devices, @usbcontrollers if @usbcontrollers;
3182     my $vga = parse_vga($conf->{vga});
3183
3184     my $qxlnum = vga_conf_has_spice($conf->{vga});
3185     $vga->{type} = 'qxl' if $qxlnum;
3186
3187     if (!$vga->{type}) {
3188         if ($arch eq 'aarch64') {
3189             $vga->{type} = 'virtio';
3190         } elsif (min_version($machine_version, 2, 9)) {
3191             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3192         } else {
3193             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3194         }
3195     }
3196
3197     # enable absolute mouse coordinates (needed by vnc)
3198     my $tablet;
3199     if (defined($conf->{tablet})) {
3200         $tablet = $conf->{tablet};
3201     } else {
3202         $tablet = $defaults->{tablet};
3203         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3204         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3205     }
3206
3207     if ($tablet) {
3208         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3209         my $kbd = print_keyboarddevice_full($conf, $arch);
3210         push @$devices, '-device', $kbd if defined($kbd);
3211     }
3212
3213     my $bootorder = device_bootorder($conf);
3214
3215     # host pci device passthrough
3216     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3217         $vmid, $conf, $devices, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3218
3219     # usb devices
3220     my $usb_dev_features = {};
3221     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3222
3223     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3224         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3225     push @$devices, @usbdevices if @usbdevices;
3226
3227     # serial devices
3228     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3229         if (my $path = $conf->{"serial$i"}) {
3230             if ($path eq 'socket') {
3231                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3232                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3233                 # On aarch64, serial0 is the UART device. Qemu only allows
3234                 # connecting UART devices via the '-serial' command line, as
3235                 # the device has a fixed slot on the hardware...
3236                 if ($arch eq 'aarch64' && $i == 0) {
3237                     push @$devices, '-serial', "chardev:serial$i";
3238                 } else {
3239                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3240                 }
3241             } else {
3242                 die "no such serial device\n" if ! -c $path;
3243                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3244                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3245             }
3246         }
3247     }
3248
3249     # parallel devices
3250     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3251         if (my $path = $conf->{"parallel$i"}) {
3252             die "no such parallel device\n" if ! -c $path;
3253             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3254             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3255             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3256         }
3257     }
3258
3259     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3260         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3261         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3262         push @$devices, @$audio_devs;
3263     }
3264
3265     my $sockets = 1;
3266     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3267     $sockets = $conf->{sockets} if  $conf->{sockets};
3268
3269     my $cores = $conf->{cores} || 1;
3270
3271     my $maxcpus = $sockets * $cores;
3272
3273     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3274
3275     my $allowed_vcpus = $cpuinfo->{cpus};
3276
3277     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3278         if ($allowed_vcpus < $maxcpus);
3279
3280     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3281
3282         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3283         for (my $i = 2; $i <= $vcpus; $i++)  {
3284             my $cpustr = print_cpu_device($conf,$i);
3285             push @$cmd, '-device', $cpustr;
3286         }
3287
3288     } else {
3289
3290         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3291     }
3292     push @$cmd, '-nodefaults';
3293
3294     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3295
3296     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3297
3298     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3299
3300     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3301         push @$devices, '-device', print_vga_device(
3302             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3303         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3304         push @$cmd,  '-vnc', "unix:$socket,password";
3305     } else {
3306         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3307         push @$cmd, '-nographic';
3308     }
3309
3310     # time drift fix
3311     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3312     my $useLocaltime = $conf->{localtime};
3313
3314     if ($winversion >= 5) { # windows
3315         $useLocaltime = 1 if !defined($conf->{localtime});
3316
3317         # use time drift fix when acpi is enabled
3318         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3319             $tdf = 1 if !defined($conf->{tdf});
3320         }
3321     }
3322
3323     if ($winversion >= 6) {
3324         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3325         push @$cmd, '-no-hpet';
3326     }
3327
3328     push @$rtcFlags, 'driftfix=slew' if $tdf;
3329
3330     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3331         push @$rtcFlags, "base=$conf->{startdate}";
3332     } elsif ($useLocaltime) {
3333         push @$rtcFlags, 'base=localtime';
3334     }
3335
3336     if ($forcecpu) {
3337         push @$cmd, '-cpu', $forcecpu;
3338     } else {
3339         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3340     }
3341
3342     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3343
3344     push @$cmd, '-S' if $conf->{freeze};
3345
3346     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3347
3348     my $guest_agent = parse_guest_agent($conf);
3349
3350     if ($guest_agent->{enabled}) {
3351         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3352         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3353
3354         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3355             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3356             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3357             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3358         } elsif ($guest_agent->{type} eq 'isa') {
3359             push @$devices, '-device', "isa-serial,chardev=qga0";
3360         }
3361     }
3362
3363     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3364     if ($rng && $version_guard->(4, 1, 2)) {
3365         check_rng_source($rng->{source});
3366
3367         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3368         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3369         my $limiter_str = "";
3370         if ($max_bytes) {
3371             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3372         }
3373
3374         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3375         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3376         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3377     }
3378
3379     my $spice_port;
3380
3381     if ($qxlnum) {
3382         if ($qxlnum > 1) {
3383             if ($winversion){
3384                 for (my $i = 1; $i < $qxlnum; $i++){
3385                     push @$devices, '-device', print_vga_device(
3386                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3387                 }
3388             } else {
3389                 # assume other OS works like Linux
3390                 my ($ram, $vram) = ("134217728", "67108864");
3391                 if ($vga->{memory}) {
3392                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3393                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3394                 }
3395                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3396                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3397             }
3398         }
3399
3400         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3401
3402         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3403         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3404         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3405
3406         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3407         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3408         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3409
3410         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3411         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3412
3413         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3414         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3415         if ($spice_enhancement->{foldersharing}) {
3416             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3417             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3418         }
3419
3420         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3421         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3422             if $spice_enhancement->{videostreaming};
3423
3424         push @$devices, '-spice', "$spice_opts";
3425     }
3426
3427     # enable balloon by default, unless explicitly disabled
3428     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3429         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3430         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3431     }
3432
3433     if ($conf->{watchdog}) {
3434         my $wdopts = parse_watchdog($conf->{watchdog});
3435         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3436         my $watchdog = $wdopts->{model} || 'i6300esb';
3437         push @$devices, '-device', "$watchdog$pciaddr";
3438         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3439     }
3440
3441     my $vollist = [];
3442     my $scsicontroller = {};
3443     my $ahcicontroller = {};
3444     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3445
3446     # Add iscsi initiator name if available
3447     if (my $initiator = get_initiator_name()) {
3448         push @$devices, '-iscsi', "initiator-name=$initiator";
3449     }
3450
3451     PVE::QemuConfig->foreach_volume($conf, sub {
3452         my ($ds, $drive) = @_;
3453
3454         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3455             push @$vollist, $drive->{file};
3456         }
3457
3458         # ignore efidisk here, already added in bios/fw handling code above
3459         return if $drive->{interface} eq 'efidisk';
3460
3461         $use_virtio = 1 if $ds =~ m/^virtio/;
3462
3463         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3464
3465         if ($drive->{interface} eq 'virtio'){
3466            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3467         }
3468
3469         if ($drive->{interface} eq 'scsi') {
3470
3471             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3472
3473             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3474                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3475
3476             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3477             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3478
3479             my $iothread = '';
3480             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3481                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3482                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3483             } elsif ($drive->{iothread}) {
3484                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3485             }
3486
3487             my $queues = '';
3488             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3489                 $queues = ",num_queues=$drive->{queues}";
3490             }
3491
3492             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3493                 if !$scsicontroller->{$controller};
3494             $scsicontroller->{$controller}=1;
3495         }
3496
3497         if ($drive->{interface} eq 'sata') {
3498             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3499             $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3500             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3501                 if !$ahcicontroller->{$controller};
3502             $ahcicontroller->{$controller}=1;
3503         }
3504
3505         my $drive_cmd = print_drive_commandline_full($storecfg, $vmid, $drive);
3506         $drive_cmd .= ',readonly' if PVE::QemuConfig->is_template($conf);
3507
3508         push @$devices, '-drive',$drive_cmd;
3509         push @$devices, '-device', print_drivedevice_full(
3510             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3511     });
3512
3513     for (my $i = 0; $i < $MAX_NETS; $i++) {
3514         my $netname = "net$i";
3515
3516         next if !$conf->{$netname};
3517         my $d = parse_net($conf->{$netname});
3518         next if !$d;
3519
3520         $use_virtio = 1 if $d->{model} eq 'virtio';
3521
3522         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
3523
3524         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
3525         push @$devices, '-netdev', $netdevfull;
3526
3527         my $netdevicefull = print_netdevice_full(
3528             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
3529
3530         push @$devices, '-device', $netdevicefull;
3531     }
3532
3533     if ($conf->{ivshmem}) {
3534         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3535
3536         my $bus;
3537         if ($q35) {
3538             $bus = print_pcie_addr("ivshmem");
3539         } else {
3540             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3541         }
3542
3543         my $ivshmem_name = $ivshmem->{name} // $vmid;
3544         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3545
3546         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3547         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
3548             .",size=$ivshmem->{size}M";
3549     }
3550
3551     # pci.4 is nested in pci.1
3552     $bridges->{1} = 1 if $bridges->{4};
3553
3554     if (!$q35) {
3555         # add pci bridges
3556         if (min_version($machine_version, 2, 3)) {
3557            $bridges->{1} = 1;
3558            $bridges->{2} = 1;
3559         }
3560
3561         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3562
3563     }
3564
3565     for my $k (sort {$b cmp $a} keys %$bridges) {
3566         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3567
3568         my $k_name = $k;
3569         if ($k == 2 && $legacy_igd) {
3570             $k_name = "$k-igd";
3571         }
3572         $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
3573
3574         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3575         if ($q35) {
3576             # add after -readconfig pve-q35.cfg
3577             splice @$devices, 2, 0, '-device', $devstr;
3578         } else {
3579             unshift @$devices, '-device', $devstr if $k > 0;
3580         }
3581     }
3582
3583     if (!$kvm) {
3584         push @$machineFlags, 'accel=tcg';
3585     }
3586
3587     my $machine_type_min = $machine_type;
3588     if ($add_pve_version) {
3589         $machine_type_min =~ s/\+pve\d+$//;
3590         $machine_type_min .= "+pve$required_pve_version";
3591     }
3592     push @$machineFlags, "type=${machine_type_min}";
3593
3594     push @$cmd, @$devices;
3595     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
3596     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
3597     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
3598
3599     if (my $vmstate = $conf->{vmstate}) {
3600         my $statepath = PVE::Storage::path($storecfg, $vmstate);
3601         push @$vollist, $vmstate;
3602         push @$cmd, '-loadstate', $statepath;
3603         print "activating and using '$vmstate' as vmstate\n";
3604     }
3605
3606     # add custom args
3607     if ($conf->{args}) {
3608         my $aa = PVE::Tools::split_args($conf->{args});
3609         push @$cmd, @$aa;
3610     }
3611
3612     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3613 }
3614
3615 sub check_rng_source {
3616     my ($source) = @_;
3617
3618     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3619     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
3620         if ! -e $source;
3621
3622     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
3623     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
3624         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
3625         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
3626         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
3627             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
3628             ." to the host.\n";
3629     }
3630 }
3631
3632 sub spice_port {
3633     my ($vmid) = @_;
3634
3635     my $res = mon_cmd($vmid, 'query-spice');
3636
3637     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3638 }
3639
3640 sub vm_devices_list {
3641     my ($vmid) = @_;
3642
3643     my $res = mon_cmd($vmid, 'query-pci');
3644     my $devices_to_check = [];
3645     my $devices = {};
3646     foreach my $pcibus (@$res) {
3647         push @$devices_to_check, @{$pcibus->{devices}},
3648     }
3649
3650     while (@$devices_to_check) {
3651         my $to_check = [];
3652         for my $d (@$devices_to_check) {
3653             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3654             next if !$d->{'pci_bridge'};
3655
3656             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3657             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3658         }
3659         $devices_to_check = $to_check;
3660     }
3661
3662     my $resblock = mon_cmd($vmid, 'query-block');
3663     foreach my $block (@$resblock) {
3664         if($block->{device} =~ m/^drive-(\S+)/){
3665                 $devices->{$1} = 1;
3666         }
3667     }
3668
3669     my $resmice = mon_cmd($vmid, 'query-mice');
3670     foreach my $mice (@$resmice) {
3671         if ($mice->{name} eq 'QEMU HID Tablet') {
3672             $devices->{tablet} = 1;
3673             last;
3674         }
3675     }
3676
3677     # for usb devices there is no query-usb
3678     # but we can iterate over the entries in
3679     # qom-list path=/machine/peripheral
3680     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3681     foreach my $per (@$resperipheral) {
3682         if ($per->{name} =~ m/^usb\d+$/) {
3683             $devices->{$per->{name}} = 1;
3684         }
3685     }
3686
3687     return $devices;
3688 }
3689
3690 sub vm_deviceplug {
3691     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3692
3693     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3694
3695     my $devices_list = vm_devices_list($vmid);
3696     return 1 if defined($devices_list->{$deviceid});
3697
3698     # add PCI bridge if we need it for the device
3699     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
3700
3701     if ($deviceid eq 'tablet') {
3702
3703         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
3704
3705     } elsif ($deviceid eq 'keyboard') {
3706
3707         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
3708
3709     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3710
3711         die "usb hotplug currently not reliable\n";
3712         # since we can't reliably hot unplug all added usb devices
3713         # and usb passthrough disables live migration
3714         # we disable usb hotplugging for now
3715         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3716
3717     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3718
3719         qemu_iothread_add($vmid, $deviceid, $device);
3720
3721         qemu_driveadd($storecfg, $vmid, $device);
3722         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
3723
3724         qemu_deviceadd($vmid, $devicefull);
3725         eval { qemu_deviceaddverify($vmid, $deviceid); };
3726         if (my $err = $@) {
3727             eval { qemu_drivedel($vmid, $deviceid); };
3728             warn $@ if $@;
3729             die $err;
3730         }
3731
3732     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3733
3734
3735         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3736         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3737         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3738
3739         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3740
3741         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3742             qemu_iothread_add($vmid, $deviceid, $device);
3743             $devicefull .= ",iothread=iothread-$deviceid";
3744         }
3745
3746         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3747             $devicefull .= ",num_queues=$device->{queues}";
3748         }
3749
3750         qemu_deviceadd($vmid, $devicefull);
3751         qemu_deviceaddverify($vmid, $deviceid);
3752
3753     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3754
3755         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
3756         qemu_driveadd($storecfg, $vmid, $device);
3757
3758         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
3759         eval { qemu_deviceadd($vmid, $devicefull); };
3760         if (my $err = $@) {
3761             eval { qemu_drivedel($vmid, $deviceid); };
3762             warn $@ if $@;
3763             die $err;
3764         }
3765
3766     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3767
3768         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
3769
3770         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
3771         my $use_old_bios_files = undef;
3772         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3773
3774         my $netdevicefull = print_netdevice_full(
3775             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
3776         qemu_deviceadd($vmid, $netdevicefull);
3777         eval {
3778             qemu_deviceaddverify($vmid, $deviceid);
3779             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
3780         };
3781         if (my $err = $@) {
3782             eval { qemu_netdevdel($vmid, $deviceid); };
3783             warn $@ if $@;
3784             die $err;
3785         }
3786
3787     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3788
3789         my $bridgeid = $2;
3790         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3791         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3792
3793         qemu_deviceadd($vmid, $devicefull);
3794         qemu_deviceaddverify($vmid, $deviceid);
3795
3796     } else {
3797         die "can't hotplug device '$deviceid'\n";
3798     }
3799
3800     return 1;
3801 }
3802
3803 # fixme: this should raise exceptions on error!
3804 sub vm_deviceunplug {
3805     my ($vmid, $conf, $deviceid) = @_;
3806
3807     my $devices_list = vm_devices_list($vmid);
3808     return 1 if !defined($devices_list->{$deviceid});
3809
3810     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
3811     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
3812
3813     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
3814
3815         qemu_devicedel($vmid, $deviceid);
3816
3817     } elsif ($deviceid =~ m/^usb\d+$/) {
3818
3819         die "usb hotplug currently not reliable\n";
3820         # when unplugging usb devices this way,
3821         # there may be remaining usb controllers/hubs
3822         # so we disable it for now
3823         qemu_devicedel($vmid, $deviceid);
3824         qemu_devicedelverify($vmid, $deviceid);
3825
3826     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3827
3828         qemu_devicedel($vmid, $deviceid);
3829         qemu_devicedelverify($vmid, $deviceid);
3830         qemu_drivedel($vmid, $deviceid);
3831         qemu_iothread_del($conf, $vmid, $deviceid);
3832
3833     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3834
3835         qemu_devicedel($vmid, $deviceid);
3836         qemu_devicedelverify($vmid, $deviceid);
3837         qemu_iothread_del($conf, $vmid, $deviceid);
3838
3839     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3840
3841         qemu_devicedel($vmid, $deviceid);
3842         qemu_drivedel($vmid, $deviceid);
3843         qemu_deletescsihw($conf, $vmid, $deviceid);
3844
3845     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3846
3847         qemu_devicedel($vmid, $deviceid);
3848         qemu_devicedelverify($vmid, $deviceid);
3849         qemu_netdevdel($vmid, $deviceid);
3850
3851     } else {
3852         die "can't unplug device '$deviceid'\n";
3853     }
3854
3855     return 1;
3856 }
3857
3858 sub qemu_deviceadd {
3859     my ($vmid, $devicefull) = @_;
3860
3861     $devicefull = "driver=".$devicefull;
3862     my %options =  split(/[=,]/, $devicefull);
3863
3864     mon_cmd($vmid, "device_add" , %options);
3865 }
3866
3867 sub qemu_devicedel {
3868     my ($vmid, $deviceid) = @_;
3869
3870     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
3871 }
3872
3873 sub qemu_iothread_add {
3874     my($vmid, $deviceid, $device) = @_;
3875
3876     if ($device->{iothread}) {
3877         my $iothreads = vm_iothreads_list($vmid);
3878         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3879     }
3880 }
3881
3882 sub qemu_iothread_del {
3883     my($conf, $vmid, $deviceid) = @_;
3884
3885     my $confid = $deviceid;
3886     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
3887         $confid = 'scsi' . $1;
3888     }
3889     my $device = parse_drive($confid, $conf->{$confid});
3890     if ($device->{iothread}) {
3891         my $iothreads = vm_iothreads_list($vmid);
3892         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3893     }
3894 }
3895
3896 sub qemu_objectadd {
3897     my($vmid, $objectid, $qomtype) = @_;
3898
3899     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3900
3901     return 1;
3902 }
3903
3904 sub qemu_objectdel {
3905     my($vmid, $objectid) = @_;
3906
3907     mon_cmd($vmid, "object-del", id => $objectid);
3908
3909     return 1;
3910 }
3911
3912 sub qemu_driveadd {
3913     my ($storecfg, $vmid, $device) = @_;
3914
3915     my $drive = print_drive_commandline_full($storecfg, $vmid, $device);
3916     $drive =~ s/\\/\\\\/g;
3917     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
3918
3919     # If the command succeeds qemu prints: "OK"
3920     return 1 if $ret =~ m/OK/s;
3921
3922     die "adding drive failed: $ret\n";
3923 }
3924
3925 sub qemu_drivedel {
3926     my($vmid, $deviceid) = @_;
3927
3928     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
3929     $ret =~ s/^\s+//;
3930
3931     return 1 if $ret eq "";
3932
3933     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3934     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3935
3936     die "deleting drive $deviceid failed : $ret\n";
3937 }
3938
3939 sub qemu_deviceaddverify {
3940     my ($vmid, $deviceid) = @_;
3941
3942     for (my $i = 0; $i <= 5; $i++) {
3943          my $devices_list = vm_devices_list($vmid);
3944          return 1 if defined($devices_list->{$deviceid});
3945          sleep 1;
3946     }
3947
3948     die "error on hotplug device '$deviceid'\n";
3949 }
3950
3951
3952 sub qemu_devicedelverify {
3953     my ($vmid, $deviceid) = @_;
3954
3955     # need to verify that the device is correctly removed as device_del
3956     # is async and empty return is not reliable
3957
3958     for (my $i = 0; $i <= 5; $i++) {
3959          my $devices_list = vm_devices_list($vmid);
3960          return 1 if !defined($devices_list->{$deviceid});
3961          sleep 1;
3962     }
3963
3964     die "error on hot-unplugging device '$deviceid'\n";
3965 }
3966
3967 sub qemu_findorcreatescsihw {
3968     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
3969
3970     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3971
3972     my $scsihwid="$controller_prefix$controller";
3973     my $devices_list = vm_devices_list($vmid);
3974
3975     if(!defined($devices_list->{$scsihwid})) {
3976         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
3977     }
3978
3979     return 1;
3980 }
3981
3982 sub qemu_deletescsihw {
3983     my ($conf, $vmid, $opt) = @_;
3984
3985     my $device = parse_drive($opt, $conf->{$opt});
3986
3987     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3988         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3989         return 1;
3990     }
3991
3992     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3993
3994     my $devices_list = vm_devices_list($vmid);
3995     foreach my $opt (keys %{$devices_list}) {
3996         if (is_valid_drivename($opt)) {
3997             my $drive = parse_drive($opt, $conf->{$opt});
3998             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3999                 return 1;
4000             }
4001         }
4002     }
4003
4004     my $scsihwid="scsihw$controller";
4005
4006     vm_deviceunplug($vmid, $conf, $scsihwid);
4007
4008     return 1;
4009 }
4010
4011 sub qemu_add_pci_bridge {
4012     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4013
4014     my $bridges = {};
4015
4016     my $bridgeid;
4017
4018     print_pci_addr($device, $bridges, $arch, $machine_type);
4019
4020     while (my ($k, $v) = each %$bridges) {
4021         $bridgeid = $k;
4022     }
4023     return 1 if !defined($bridgeid) || $bridgeid < 1;
4024
4025     my $bridge = "pci.$bridgeid";
4026     my $devices_list = vm_devices_list($vmid);
4027
4028     if (!defined($devices_list->{$bridge})) {
4029         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4030     }
4031
4032     return 1;
4033 }
4034
4035 sub qemu_set_link_status {
4036     my ($vmid, $device, $up) = @_;
4037
4038     mon_cmd($vmid, "set_link", name => $device,
4039                up => $up ? JSON::true : JSON::false);
4040 }
4041
4042 sub qemu_netdevadd {
4043     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4044
4045     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4046     my %options =  split(/[=,]/, $netdev);
4047
4048     if (defined(my $vhost = $options{vhost})) {
4049         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4050     }
4051
4052     if (defined(my $queues = $options{queues})) {
4053         $options{queues} = $queues + 0;
4054     }
4055
4056     mon_cmd($vmid, "netdev_add",  %options);
4057     return 1;
4058 }
4059
4060 sub qemu_netdevdel {
4061     my ($vmid, $deviceid) = @_;
4062
4063     mon_cmd($vmid, "netdev_del", id => $deviceid);
4064 }
4065
4066 sub qemu_usb_hotplug {
4067     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4068
4069     return if !$device;
4070
4071     # remove the old one first
4072     vm_deviceunplug($vmid, $conf, $deviceid);
4073
4074     # check if xhci controller is necessary and available
4075     if ($device->{usb3}) {
4076
4077         my $devicelist = vm_devices_list($vmid);
4078
4079         if (!$devicelist->{xhci}) {
4080             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4081             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4082         }
4083     }
4084     my $d = parse_usb_device($device->{host});
4085     $d->{usb3} = $device->{usb3};
4086
4087     # add the new one
4088     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4089 }
4090
4091 sub qemu_cpu_hotplug {
4092     my ($vmid, $conf, $vcpus) = @_;
4093
4094     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4095
4096     my $sockets = 1;
4097     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4098     $sockets = $conf->{sockets} if  $conf->{sockets};
4099     my $cores = $conf->{cores} || 1;
4100     my $maxcpus = $sockets * $cores;
4101
4102     $vcpus = $maxcpus if !$vcpus;
4103
4104     die "you can't add more vcpus than maxcpus\n"
4105         if $vcpus > $maxcpus;
4106
4107     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4108
4109     if ($vcpus < $currentvcpus) {
4110
4111         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4112
4113             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4114                 qemu_devicedel($vmid, "cpu$i");
4115                 my $retry = 0;
4116                 my $currentrunningvcpus = undef;
4117                 while (1) {
4118                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4119                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4120                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4121                     $retry++;
4122                     sleep 1;
4123                 }
4124                 #update conf after each succesfull cpu unplug
4125                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4126                 PVE::QemuConfig->write_config($vmid, $conf);
4127             }
4128         } else {
4129             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4130         }
4131
4132         return;
4133     }
4134
4135     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4136     die "vcpus in running vm does not match its configuration\n"
4137         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4138
4139     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4140
4141         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4142             my $cpustr = print_cpu_device($conf, $i);
4143             qemu_deviceadd($vmid, $cpustr);
4144
4145             my $retry = 0;
4146             my $currentrunningvcpus = undef;
4147             while (1) {
4148                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4149                 last if scalar(@{$currentrunningvcpus}) == $i;
4150                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4151                 sleep 1;
4152                 $retry++;
4153             }
4154             #update conf after each succesfull cpu hotplug
4155             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4156             PVE::QemuConfig->write_config($vmid, $conf);
4157         }
4158     } else {
4159
4160         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4161             mon_cmd($vmid, "cpu-add", id => int($i));
4162         }
4163     }
4164 }
4165
4166 sub qemu_block_set_io_throttle {
4167     my ($vmid, $deviceid,
4168         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4169         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4170         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4171         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4172
4173     return if !check_running($vmid) ;
4174
4175     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4176         bps => int($bps),
4177         bps_rd => int($bps_rd),
4178         bps_wr => int($bps_wr),
4179         iops => int($iops),
4180         iops_rd => int($iops_rd),
4181         iops_wr => int($iops_wr),
4182         bps_max => int($bps_max),
4183         bps_rd_max => int($bps_rd_max),
4184         bps_wr_max => int($bps_wr_max),
4185         iops_max => int($iops_max),
4186         iops_rd_max => int($iops_rd_max),
4187         iops_wr_max => int($iops_wr_max),
4188         bps_max_length => int($bps_max_length),
4189         bps_rd_max_length => int($bps_rd_max_length),
4190         bps_wr_max_length => int($bps_wr_max_length),
4191         iops_max_length => int($iops_max_length),
4192         iops_rd_max_length => int($iops_rd_max_length),
4193         iops_wr_max_length => int($iops_wr_max_length),
4194     );
4195
4196 }
4197
4198 # old code, only used to shutdown old VM after update
4199 sub __read_avail {
4200     my ($fh, $timeout) = @_;
4201
4202     my $sel = new IO::Select;
4203     $sel->add($fh);
4204
4205     my $res = '';
4206     my $buf;
4207
4208     my @ready;
4209     while (scalar (@ready = $sel->can_read($timeout))) {
4210         my $count;
4211         if ($count = $fh->sysread($buf, 8192)) {
4212             if ($buf =~ /^(.*)\(qemu\) $/s) {
4213                 $res .= $1;
4214                 last;
4215             } else {
4216                 $res .= $buf;
4217             }
4218         } else {
4219             if (!defined($count)) {
4220                 die "$!\n";
4221             }
4222             last;
4223         }
4224     }
4225
4226     die "monitor read timeout\n" if !scalar(@ready);
4227
4228     return $res;
4229 }
4230
4231 sub qemu_block_resize {
4232     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4233
4234     my $running = check_running($vmid);
4235
4236     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4237
4238     return if !$running;
4239
4240     my $padding = (1024 - $size % 1024) % 1024;
4241     $size = $size + $padding;
4242
4243     mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4244
4245 }
4246
4247 sub qemu_volume_snapshot {
4248     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4249
4250     my $running = check_running($vmid);
4251
4252     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4253         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4254     } else {
4255         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4256     }
4257 }
4258
4259 sub qemu_volume_snapshot_delete {
4260     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4261
4262     my $running = check_running($vmid);
4263
4264     if($running) {
4265
4266         $running = undef;
4267         my $conf = PVE::QemuConfig->load_config($vmid);
4268         PVE::QemuConfig->foreach_volume($conf, sub {
4269             my ($ds, $drive) = @_;
4270             $running = 1 if $drive->{file} eq $volid;
4271         });
4272     }
4273
4274     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4275         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4276     } else {
4277         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4278     }
4279 }
4280
4281 sub set_migration_caps {
4282     my ($vmid) = @_;
4283
4284     my $cap_ref = [];
4285
4286     my $enabled_cap = {
4287         "auto-converge" => 1,
4288         "xbzrle" => 1,
4289         "x-rdma-pin-all" => 0,
4290         "zero-blocks" => 0,
4291         "compress" => 0
4292     };
4293
4294     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4295
4296     for my $supported_capability (@$supported_capabilities) {
4297         push @$cap_ref, {
4298             capability => $supported_capability->{capability},
4299             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4300         };
4301     }
4302
4303     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4304 }
4305
4306 sub foreach_volid {
4307     my ($conf, $func, @param) = @_;
4308
4309     my $volhash = {};
4310
4311     my $test_volid = sub {
4312         my ($key, $drive, $snapname) = @_;
4313
4314         my $volid = $drive->{file};
4315         return if !$volid;
4316
4317         $volhash->{$volid}->{cdrom} //= 1;
4318         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4319
4320         my $replicate = $drive->{replicate} // 1;
4321         $volhash->{$volid}->{replicate} //= 0;
4322         $volhash->{$volid}->{replicate} = 1 if $replicate;
4323
4324         $volhash->{$volid}->{shared} //= 0;
4325         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4326
4327         $volhash->{$volid}->{referenced_in_config} //= 0;
4328         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4329
4330         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4331             if defined($snapname);
4332
4333         my $size = $drive->{size};
4334         $volhash->{$volid}->{size} //= $size if $size;
4335
4336         $volhash->{$volid}->{is_vmstate} //= 0;
4337         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4338
4339         $volhash->{$volid}->{is_unused} //= 0;
4340         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4341     };
4342
4343     my $include_opts = {
4344         extra_keys => ['vmstate'],
4345         include_unused => 1,
4346     };
4347
4348     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4349     foreach my $snapname (keys %{$conf->{snapshots}}) {
4350         my $snap = $conf->{snapshots}->{$snapname};
4351         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4352     }
4353
4354     foreach my $volid (keys %$volhash) {
4355         &$func($volid, $volhash->{$volid}, @param);
4356     }
4357 }
4358
4359 my $fast_plug_option = {
4360     'lock' => 1,
4361     'name' => 1,
4362     'onboot' => 1,
4363     'shares' => 1,
4364     'startup' => 1,
4365     'description' => 1,
4366     'protection' => 1,
4367     'vmstatestorage' => 1,
4368     'hookscript' => 1,
4369     'tags' => 1,
4370 };
4371
4372 # hotplug changes in [PENDING]
4373 # $selection hash can be used to only apply specified options, for
4374 # example: { cores => 1 } (only apply changed 'cores')
4375 # $errors ref is used to return error messages
4376 sub vmconfig_hotplug_pending {
4377     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4378
4379     my $defaults = load_defaults();
4380     my $arch = get_vm_arch($conf);
4381     my $machine_type = get_vm_machine($conf, undef, $arch);
4382
4383     # commit values which do not have any impact on running VM first
4384     # Note: those option cannot raise errors, we we do not care about
4385     # $selection and always apply them.
4386
4387     my $add_error = sub {
4388         my ($opt, $msg) = @_;
4389         $errors->{$opt} = "hotplug problem - $msg";
4390     };
4391
4392     my $changes = 0;
4393     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4394         if ($fast_plug_option->{$opt}) {
4395             $conf->{$opt} = $conf->{pending}->{$opt};
4396             delete $conf->{pending}->{$opt};
4397             $changes = 1;
4398         }
4399     }
4400
4401     if ($changes) {
4402         PVE::QemuConfig->write_config($vmid, $conf);
4403     }
4404
4405     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4406
4407     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4408     foreach my $opt (sort keys %$pending_delete_hash) {
4409         next if $selection && !$selection->{$opt};
4410         my $force = $pending_delete_hash->{$opt}->{force};
4411         eval {
4412             if ($opt eq 'hotplug') {
4413                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4414             } elsif ($opt eq 'tablet') {
4415                 die "skip\n" if !$hotplug_features->{usb};
4416                 if ($defaults->{tablet}) {
4417                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4418                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4419                         if $arch eq 'aarch64';
4420                 } else {
4421                     vm_deviceunplug($vmid, $conf, 'tablet');
4422                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4423                 }
4424             } elsif ($opt =~ m/^usb\d+/) {
4425                 die "skip\n";
4426                 # since we cannot reliably hot unplug usb devices
4427                 # we are disabling it
4428                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4429                 vm_deviceunplug($vmid, $conf, $opt);
4430             } elsif ($opt eq 'vcpus') {
4431                 die "skip\n" if !$hotplug_features->{cpu};
4432                 qemu_cpu_hotplug($vmid, $conf, undef);
4433             } elsif ($opt eq 'balloon') {
4434                 # enable balloon device is not hotpluggable
4435                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4436                 # here we reset the ballooning value to memory
4437                 my $balloon = $conf->{memory} || $defaults->{memory};
4438                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4439             } elsif ($fast_plug_option->{$opt}) {
4440                 # do nothing
4441             } elsif ($opt =~ m/^net(\d+)$/) {
4442                 die "skip\n" if !$hotplug_features->{network};
4443                 vm_deviceunplug($vmid, $conf, $opt);
4444             } elsif (is_valid_drivename($opt)) {
4445                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4446                 vm_deviceunplug($vmid, $conf, $opt);
4447                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4448             } elsif ($opt =~ m/^memory$/) {
4449                 die "skip\n" if !$hotplug_features->{memory};
4450                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4451             } elsif ($opt eq 'cpuunits') {
4452                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4453             } elsif ($opt eq 'cpulimit') {
4454                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4455             } else {
4456                 die "skip\n";
4457             }
4458         };
4459         if (my $err = $@) {
4460             &$add_error($opt, $err) if $err ne "skip\n";
4461         } else {
4462             delete $conf->{$opt};
4463             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4464         }
4465     }
4466
4467     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4468     $apply_pending_cloudinit = sub {
4469         return if $apply_pending_cloudinit_done; # once is enough
4470         $apply_pending_cloudinit_done = 1; # once is enough
4471
4472         my ($key, $value) = @_;
4473
4474         my @cloudinit_opts = keys %$confdesc_cloudinit;
4475         foreach my $opt (keys %{$conf->{pending}}) {
4476             next if !grep { $_ eq $opt } @cloudinit_opts;
4477             $conf->{$opt} = delete $conf->{pending}->{$opt};
4478         }
4479
4480         my $new_conf = { %$conf };
4481         $new_conf->{$key} = $value;
4482         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4483     };
4484
4485     foreach my $opt (keys %{$conf->{pending}}) {
4486         next if $selection && !$selection->{$opt};
4487         my $value = $conf->{pending}->{$opt};
4488         eval {
4489             if ($opt eq 'hotplug') {
4490                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4491             } elsif ($opt eq 'tablet') {
4492                 die "skip\n" if !$hotplug_features->{usb};
4493                 if ($value == 1) {
4494                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4495                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4496                         if $arch eq 'aarch64';
4497                 } elsif ($value == 0) {
4498                     vm_deviceunplug($vmid, $conf, 'tablet');
4499                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4500                 }
4501             } elsif ($opt =~ m/^usb\d+$/) {
4502                 die "skip\n";
4503                 # since we cannot reliably hot unplug usb devices
4504                 # we are disabling it
4505                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4506                 my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4507                 die "skip\n" if !$d;
4508                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4509             } elsif ($opt eq 'vcpus') {
4510                 die "skip\n" if !$hotplug_features->{cpu};
4511                 qemu_cpu_hotplug($vmid, $conf, $value);
4512             } elsif ($opt eq 'balloon') {
4513                 # enable/disable balloning device is not hotpluggable
4514                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4515                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4516                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4517
4518                 # allow manual ballooning if shares is set to zero
4519                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4520                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4521                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4522                 }
4523             } elsif ($opt =~ m/^net(\d+)$/) {
4524                 # some changes can be done without hotplug
4525                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4526                                     $vmid, $opt, $value, $arch, $machine_type);
4527             } elsif (is_valid_drivename($opt)) {
4528                 die "skip\n" if $opt eq 'efidisk0';
4529                 # some changes can be done without hotplug
4530                 my $drive = parse_drive($opt, $value);
4531                 if (drive_is_cloudinit($drive)) {
4532                     &$apply_pending_cloudinit($opt, $value);
4533                 }
4534                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4535                                      $vmid, $opt, $value, $arch, $machine_type);
4536             } elsif ($opt =~ m/^memory$/) { #dimms
4537                 die "skip\n" if !$hotplug_features->{memory};
4538                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4539             } elsif ($opt eq 'cpuunits') {
4540                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4541             } elsif ($opt eq 'cpulimit') {
4542                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4543                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4544             } else {
4545                 die "skip\n";  # skip non-hot-pluggable options
4546             }
4547         };
4548         if (my $err = $@) {
4549             &$add_error($opt, $err) if $err ne "skip\n";
4550         } else {
4551             $conf->{$opt} = $value;
4552             delete $conf->{pending}->{$opt};
4553         }
4554     }
4555
4556     PVE::QemuConfig->write_config($vmid, $conf);
4557 }
4558
4559 sub try_deallocate_drive {
4560     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4561
4562     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4563         my $volid = $drive->{file};
4564         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4565             my $sid = PVE::Storage::parse_volume_id($volid);
4566             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4567
4568             # check if the disk is really unused
4569             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4570                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4571             PVE::Storage::vdisk_free($storecfg, $volid);
4572             return 1;
4573         } else {
4574             # If vm is not owner of this disk remove from config
4575             return 1;
4576         }
4577     }
4578
4579     return;
4580 }
4581
4582 sub vmconfig_delete_or_detach_drive {
4583     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4584
4585     my $drive = parse_drive($opt, $conf->{$opt});
4586
4587     my $rpcenv = PVE::RPCEnvironment::get();
4588     my $authuser = $rpcenv->get_user();
4589
4590     if ($force) {
4591         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4592         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4593     } else {
4594         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4595     }
4596 }
4597
4598
4599
4600 sub vmconfig_apply_pending {
4601     my ($vmid, $conf, $storecfg, $errors) = @_;
4602
4603     my $add_apply_error = sub {
4604         my ($opt, $msg) = @_;
4605         my $err_msg = "unable to apply pending change $opt : $msg";
4606         $errors->{$opt} = $err_msg;
4607         warn $err_msg;
4608     };
4609
4610     # cold plug
4611
4612     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4613     foreach my $opt (sort keys %$pending_delete_hash) {
4614         my $force = $pending_delete_hash->{$opt}->{force};
4615         eval {
4616             if ($opt =~ m/^unused/) {
4617                 die "internal error";
4618             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4619                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4620             }
4621         };
4622         if (my $err = $@) {
4623             $add_apply_error->($opt, $err);
4624         } else {
4625             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4626             delete $conf->{$opt};
4627         }
4628     }
4629
4630     PVE::QemuConfig->cleanup_pending($conf);
4631
4632     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4633         next if $opt eq 'delete'; # just to be sure
4634         eval {
4635             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4636                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4637             }
4638         };
4639         if (my $err = $@) {
4640             $add_apply_error->($opt, $err);
4641         } else {
4642             $conf->{$opt} = delete $conf->{pending}->{$opt};
4643         }
4644     }
4645
4646     # write all changes at once to avoid unnecessary i/o
4647     PVE::QemuConfig->write_config($vmid, $conf);
4648 }
4649
4650 sub vmconfig_update_net {
4651     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4652
4653     my $newnet = parse_net($value);
4654
4655     if ($conf->{$opt}) {
4656         my $oldnet = parse_net($conf->{$opt});
4657
4658         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4659             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4660             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4661             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4662
4663             # for non online change, we try to hot-unplug
4664             die "skip\n" if !$hotplug;
4665             vm_deviceunplug($vmid, $conf, $opt);
4666         } else {
4667
4668             die "internal error" if $opt !~ m/net(\d+)/;
4669             my $iface = "tap${vmid}i$1";
4670
4671             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4672                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4673                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4674                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4675                 PVE::Network::tap_unplug($iface);
4676
4677                 if ($have_sdn) {
4678                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4679                 } else {
4680                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4681                 }
4682             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4683                 # Rate can be applied on its own but any change above needs to
4684                 # include the rate in tap_plug since OVS resets everything.
4685                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4686             }
4687
4688             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4689                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4690             }
4691
4692             return 1;
4693         }
4694     }
4695
4696     if ($hotplug) {
4697         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4698     } else {
4699         die "skip\n";
4700     }
4701 }
4702
4703 sub vmconfig_update_disk {
4704     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4705
4706     my $drive = parse_drive($opt, $value);
4707
4708     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
4709         my $media = $drive->{media} || 'disk';
4710         my $oldmedia = $old_drive->{media} || 'disk';
4711         die "unable to change media type\n" if $media ne $oldmedia;
4712
4713         if (!drive_is_cdrom($old_drive)) {
4714
4715             if ($drive->{file} ne $old_drive->{file}) {
4716
4717                 die "skip\n" if !$hotplug;
4718
4719                 # unplug and register as unused
4720                 vm_deviceunplug($vmid, $conf, $opt);
4721                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4722
4723             } else {
4724                 # update existing disk
4725
4726                 # skip non hotpluggable value
4727                 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4728                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4729                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4730                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
4731                     safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
4732                     die "skip\n";
4733                 }
4734
4735                 # apply throttle
4736                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4737                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4738                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4739                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4740                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4741                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4742                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4743                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4744                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4745                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4746                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4747                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4748                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4749                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4750                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4751                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4752                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4753                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4754
4755                     qemu_block_set_io_throttle(
4756                         $vmid,"drive-$opt",
4757                         ($drive->{mbps} || 0)*1024*1024,
4758                         ($drive->{mbps_rd} || 0)*1024*1024,
4759                         ($drive->{mbps_wr} || 0)*1024*1024,
4760                          $drive->{iops} || 0,
4761                          $drive->{iops_rd} || 0,
4762                          $drive->{iops_wr} || 0,
4763                         ($drive->{mbps_max} || 0)*1024*1024,
4764                         ($drive->{mbps_rd_max} || 0)*1024*1024,
4765                         ($drive->{mbps_wr_max} || 0)*1024*1024,
4766                          $drive->{iops_max} || 0,
4767                          $drive->{iops_rd_max} || 0,
4768                          $drive->{iops_wr_max} || 0,
4769                          $drive->{bps_max_length} || 1,
4770                          $drive->{bps_rd_max_length} || 1,
4771                          $drive->{bps_wr_max_length} || 1,
4772                          $drive->{iops_max_length} || 1,
4773                          $drive->{iops_rd_max_length} || 1,
4774                          $drive->{iops_wr_max_length} || 1,
4775                     );
4776
4777                 }
4778
4779                 return 1;
4780             }
4781
4782         } else { # cdrom
4783
4784             if ($drive->{file} eq 'none') {
4785                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4786                 if (drive_is_cloudinit($old_drive)) {
4787                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4788                 }
4789             } else {
4790                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4791
4792                 # force eject if locked
4793                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4794
4795                 if ($path) {
4796                     mon_cmd($vmid, "blockdev-change-medium",
4797                         id => "$opt", filename => "$path");
4798                 }
4799             }
4800
4801             return 1;
4802         }
4803     }
4804
4805     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4806     # hotplug new disks
4807     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4808     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
4809 }
4810
4811 # called in locked context by incoming migration
4812 sub vm_migrate_get_nbd_disks {
4813     my ($storecfg, $conf, $replicated_volumes) = @_;
4814
4815     my $local_volumes = {};
4816     PVE::QemuConfig->foreach_volume($conf, sub {
4817         my ($ds, $drive) = @_;
4818
4819         return if drive_is_cdrom($drive);
4820
4821         my $volid = $drive->{file};
4822
4823         return if !$volid;
4824
4825         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4826
4827         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4828         return if $scfg->{shared};
4829
4830         # replicated disks re-use existing state via bitmap
4831         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
4832         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
4833     });
4834     return $local_volumes;
4835 }
4836
4837 # called in locked context by incoming migration
4838 sub vm_migrate_alloc_nbd_disks {
4839     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
4840
4841     my $format = undef;
4842
4843     my $nbd = {};
4844     foreach my $opt (sort keys %$source_volumes) {
4845         my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
4846
4847         if ($use_existing) {
4848             $nbd->{$opt}->{drivestr} = print_drive($drive);
4849             $nbd->{$opt}->{volid} = $volid;
4850             $nbd->{$opt}->{replicated} = 1;
4851             next;
4852         }
4853
4854         # If a remote storage is specified and the format of the original
4855         # volume is not available there, fall back to the default format.
4856         # Otherwise use the same format as the original.
4857         if (!$storagemap->{identity}) {
4858             $storeid = map_storage($storagemap, $storeid);
4859             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4860             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4861             my $fileFormat = qemu_img_format($scfg, $volname);
4862             $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
4863         } else {
4864             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4865             $format = qemu_img_format($scfg, $volname);
4866         }
4867
4868         my $size = $drive->{size} / 1024;
4869         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
4870         my $newdrive = $drive;
4871         $newdrive->{format} = $format;
4872         $newdrive->{file} = $newvolid;
4873         my $drivestr = print_drive($newdrive);
4874         $nbd->{$opt}->{drivestr} = $drivestr;
4875         $nbd->{$opt}->{volid} = $newvolid;
4876     }
4877
4878     return $nbd;
4879 }
4880
4881 # see vm_start_nolock for parameters, additionally:
4882 # migrate_opts:
4883 #   storagemap = parsed storage map for allocating NBD disks
4884 sub vm_start {
4885     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
4886
4887     return PVE::QemuConfig->lock_config($vmid, sub {
4888         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
4889
4890         die "you can't start a vm if it's a template\n"
4891             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
4892
4893         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
4894
4895         PVE::QemuConfig->check_lock($conf)
4896             if !($params->{skiplock} || $has_suspended_lock);
4897
4898         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
4899
4900         die "VM $vmid already running\n" if check_running($vmid, undef, $migrate_opts->{migratedfrom});
4901
4902         if (my $storagemap = $migrate_opts->{storagemap}) {
4903             my $replicated = $migrate_opts->{replicated_volumes};
4904             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
4905             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
4906
4907             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
4908                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
4909             }
4910         }
4911
4912         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
4913     });
4914 }
4915
4916
4917 # params:
4918 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
4919 #   skiplock => 0/1, skip checking for config lock
4920 #   skiptemplate => 0/1, skip checking whether VM is template
4921 #   forcemachine => to force Qemu machine (rollback/migration)
4922 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
4923 #   timeout => in seconds
4924 #   paused => start VM in paused state (backup)
4925 #   resume => resume from hibernation
4926 # migrate_opts:
4927 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
4928 #   migratedfrom => source node
4929 #   spice_ticket => used for spice migration, passed via tunnel/stdin
4930 #   network => CIDR of migration network
4931 #   type => secure/insecure - tunnel over encrypted connection or plain-text
4932 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
4933 #   replicated_volumes = which volids should be re-used with bitmaps for nbd migration
4934 sub vm_start_nolock {
4935     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
4936
4937     my $statefile = $params->{statefile};
4938     my $resume = $params->{resume};
4939
4940     my $migratedfrom = $migrate_opts->{migratedfrom};
4941     my $migration_type = $migrate_opts->{type};
4942
4943     my $res = {};
4944
4945     # clean up leftover reboot request files
4946     eval { clear_reboot_request($vmid); };
4947     warn $@ if $@;
4948
4949     if (!$statefile && scalar(keys %{$conf->{pending}})) {
4950         vmconfig_apply_pending($vmid, $conf, $storecfg);
4951         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4952     }
4953
4954     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4955
4956     my $defaults = load_defaults();
4957
4958     # set environment variable useful inside network script
4959     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4960
4961     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
4962
4963     my $forcemachine = $params->{forcemachine};
4964     my $forcecpu = $params->{forcecpu};
4965     if ($resume) {
4966         # enforce machine and CPU type on suspended vm to ensure HW compatibility
4967         $forcemachine = $conf->{runningmachine};
4968         $forcecpu = $conf->{runningcpu};
4969         print "Resuming suspended VM\n";
4970     }
4971
4972     my ($cmd, $vollist, $spice_port) =
4973         config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
4974
4975     my $migration_ip;
4976     my $get_migration_ip = sub {
4977         my ($nodename) = @_;
4978
4979         return $migration_ip if defined($migration_ip);
4980
4981         my $cidr = $migrate_opts->{network};
4982
4983         if (!defined($cidr)) {
4984             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4985             $cidr = $dc_conf->{migration}->{network};
4986         }
4987
4988         if (defined($cidr)) {
4989             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
4990
4991             die "could not get IP: no address configured on local " .
4992                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
4993
4994             die "could not get IP: multiple addresses configured on local " .
4995                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
4996
4997             $migration_ip = @$ips[0];
4998         }
4999
5000         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5001             if !defined($migration_ip);
5002
5003         return $migration_ip;
5004     };
5005
5006     my $migrate_uri;
5007     if ($statefile) {
5008         if ($statefile eq 'tcp') {
5009             my $localip = "localhost";
5010             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5011             my $nodename = nodename();
5012
5013             if (!defined($migration_type)) {
5014                 if (defined($datacenterconf->{migration}->{type})) {
5015                     $migration_type = $datacenterconf->{migration}->{type};
5016                 } else {
5017                     $migration_type = 'secure';
5018                 }
5019             }
5020
5021             if ($migration_type eq 'insecure') {
5022                 $localip = $get_migration_ip->($nodename);
5023                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5024             }
5025
5026             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5027             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5028             $migrate_uri = "tcp:${localip}:${migrate_port}";
5029             push @$cmd, '-incoming', $migrate_uri;
5030             push @$cmd, '-S';
5031
5032         } elsif ($statefile eq 'unix') {
5033             # should be default for secure migrations as a ssh TCP forward
5034             # tunnel is not deterministic reliable ready and fails regurarly
5035             # to set up in time, so use UNIX socket forwards
5036             my $socket_addr = "/run/qemu-server/$vmid.migrate";
5037             unlink $socket_addr;
5038
5039             $migrate_uri = "unix:$socket_addr";
5040
5041             push @$cmd, '-incoming', $migrate_uri;
5042             push @$cmd, '-S';
5043
5044         } elsif (-e $statefile) {
5045             push @$cmd, '-loadstate', $statefile;
5046         } else {
5047             my $statepath = PVE::Storage::path($storecfg, $statefile);
5048             push @$vollist, $statefile;
5049             push @$cmd, '-loadstate', $statepath;
5050         }
5051     } elsif ($params->{paused}) {
5052         push @$cmd, '-S';
5053     }
5054
5055     # host pci devices
5056     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5057       my $d = parse_hostpci($conf->{"hostpci$i"});
5058       next if !$d;
5059       my $pcidevices = $d->{pciid};
5060       foreach my $pcidevice (@$pcidevices) {
5061             my $pciid = $pcidevice->{id};
5062
5063             my $info = PVE::SysFSTools::pci_device_info("$pciid");
5064             die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5065             die "no pci device info for device '$pciid'\n" if !$info;
5066
5067             if ($d->{mdev}) {
5068                 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5069                 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5070             } else {
5071                 die "can't unbind/bind pci group to vfio '$pciid'\n"
5072                     if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5073                 die "can't reset pci device '$pciid'\n"
5074                     if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5075             }
5076       }
5077     }
5078
5079     PVE::Storage::activate_volumes($storecfg, $vollist);
5080
5081     eval {
5082         run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5083             outfunc => sub {}, errfunc => sub {});
5084     };
5085     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5086     # timeout should be more than enough here...
5087     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5088
5089     my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5090                                               : $defaults->{cpuunits};
5091
5092     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5093     my %run_params = (
5094         timeout => $statefile ? undef : $start_timeout,
5095         umask => 0077,
5096         noerr => 1,
5097     );
5098
5099     # when migrating, prefix QEMU output so other side can pick up any
5100     # errors that might occur and show the user
5101     if ($migratedfrom) {
5102         $run_params{quiet} = 1;
5103         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5104     }
5105
5106     my %properties = (
5107         Slice => 'qemu.slice',
5108         KillMode => 'none',
5109         CPUShares => $cpuunits
5110     );
5111
5112     if (my $cpulimit = $conf->{cpulimit}) {
5113         $properties{CPUQuota} = int($cpulimit * 100);
5114     }
5115     $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5116
5117     my $run_qemu = sub {
5118         PVE::Tools::run_fork sub {
5119             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5120
5121             my $exitcode = run_command($cmd, %run_params);
5122             die "QEMU exited with code $exitcode\n" if $exitcode;
5123         };
5124     };
5125
5126     if ($conf->{hugepages}) {
5127
5128         my $code = sub {
5129             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5130             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5131
5132             PVE::QemuServer::Memory::hugepages_mount();
5133             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5134
5135             eval { $run_qemu->() };
5136             if (my $err = $@) {
5137                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5138                     if !$conf->{keephugepages};
5139                 die $err;
5140             }
5141
5142             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5143                 if !$conf->{keephugepages};
5144         };
5145         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5146
5147     } else {
5148         eval { $run_qemu->() };
5149     }
5150
5151     if (my $err = $@) {
5152         # deactivate volumes if start fails
5153         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5154         die "start failed: $err";
5155     }
5156
5157     print "migration listens on $migrate_uri\n" if $migrate_uri;
5158     $res->{migrate_uri} = $migrate_uri;
5159
5160     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5161         eval { mon_cmd($vmid, "cont"); };
5162         warn $@ if $@;
5163     }
5164
5165     #start nbd server for storage migration
5166     if (my $nbd = $migrate_opts->{nbd}) {
5167         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5168
5169         my $migrate_storage_uri;
5170         # nbd_protocol_version > 0 for unix socket support
5171         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5172             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5173             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5174             $migrate_storage_uri = "nbd:unix:$socket_path";
5175         } else {
5176             my $nodename = nodename();
5177             my $localip = $get_migration_ip->($nodename);
5178             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5179             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5180
5181             mon_cmd($vmid, "nbd-server-start", addr => {
5182                 type => 'inet',
5183                 data => {
5184                     host => "${localip}",
5185                     port => "${storage_migrate_port}",
5186                 },
5187             });
5188             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5189             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5190         }
5191
5192         $res->{migrate_storage_uri} = $migrate_storage_uri;
5193
5194         foreach my $opt (sort keys %$nbd) {
5195             my $drivestr = $nbd->{$opt}->{drivestr};
5196             my $volid = $nbd->{$opt}->{volid};
5197             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5198             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5199             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5200             print "re-using replicated volume: $opt - $volid\n"
5201                 if $nbd->{$opt}->{replicated};
5202
5203             $res->{drives}->{$opt} = $nbd->{$opt};
5204             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5205         }
5206     }
5207
5208     if ($migratedfrom) {
5209         eval {
5210             set_migration_caps($vmid);
5211         };
5212         warn $@ if $@;
5213
5214         if ($spice_port) {
5215             print "spice listens on port $spice_port\n";
5216             $res->{spice_port} = $spice_port;
5217             if ($migrate_opts->{spice_ticket}) {
5218                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5219                     $migrate_opts->{spice_ticket});
5220                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5221             }
5222         }
5223
5224     } else {
5225         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5226             if !$statefile && $conf->{balloon};
5227
5228         foreach my $opt (keys %$conf) {
5229             next if $opt !~  m/^net\d+$/;
5230             my $nicconf = parse_net($conf->{$opt});
5231             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5232         }
5233     }
5234
5235     mon_cmd($vmid, 'qom-set',
5236                 path => "machine/peripheral/balloon0",
5237                 property => "guest-stats-polling-interval",
5238                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5239
5240     if ($resume) {
5241         print "Resumed VM, removing state\n";
5242         if (my $vmstate = $conf->{vmstate}) {
5243             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5244             PVE::Storage::vdisk_free($storecfg, $vmstate);
5245         }
5246         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5247         PVE::QemuConfig->write_config($vmid, $conf);
5248     }
5249
5250     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5251
5252     return $res;
5253 }
5254
5255 sub vm_commandline {
5256     my ($storecfg, $vmid, $snapname) = @_;
5257
5258     my $conf = PVE::QemuConfig->load_config($vmid);
5259     my $forcemachine;
5260     my $forcecpu;
5261
5262     if ($snapname) {
5263         my $snapshot = $conf->{snapshots}->{$snapname};
5264         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5265
5266         # check for machine or CPU overrides in snapshot
5267         $forcemachine = $snapshot->{runningmachine};
5268         $forcecpu = $snapshot->{runningcpu};
5269
5270         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5271
5272         $conf = $snapshot;
5273     }
5274
5275     my $defaults = load_defaults();
5276
5277     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults,
5278         $forcemachine, $forcecpu);
5279
5280     return PVE::Tools::cmd2string($cmd);
5281 }
5282
5283 sub vm_reset {
5284     my ($vmid, $skiplock) = @_;
5285
5286     PVE::QemuConfig->lock_config($vmid, sub {
5287
5288         my $conf = PVE::QemuConfig->load_config($vmid);
5289
5290         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5291
5292         mon_cmd($vmid, "system_reset");
5293     });
5294 }
5295
5296 sub get_vm_volumes {
5297     my ($conf) = @_;
5298
5299     my $vollist = [];
5300     foreach_volid($conf, sub {
5301         my ($volid, $attr) = @_;
5302
5303         return if $volid =~ m|^/|;
5304
5305         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5306         return if !$sid;
5307
5308         push @$vollist, $volid;
5309     });
5310
5311     return $vollist;
5312 }
5313
5314 sub vm_stop_cleanup {
5315     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5316
5317     eval {
5318
5319         if (!$keepActive) {
5320             my $vollist = get_vm_volumes($conf);
5321             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5322         }
5323
5324         foreach my $ext (qw(mon qmp pid vnc qga)) {
5325             unlink "/var/run/qemu-server/${vmid}.$ext";
5326         }
5327
5328         if ($conf->{ivshmem}) {
5329             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5330             # just delete it for now, VMs which have this already open do not
5331             # are affected, but new VMs will get a separated one. If this
5332             # becomes an issue we either add some sort of ref-counting or just
5333             # add a "don't delete on stop" flag to the ivshmem format.
5334             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5335         }
5336
5337         foreach my $key (keys %$conf) {
5338             next if $key !~ m/^hostpci(\d+)$/;
5339             my $hostpciindex = $1;
5340             my $d = parse_hostpci($conf->{$key});
5341             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5342
5343             foreach my $pci (@{$d->{pciid}}) {
5344                 my $pciid = $pci->{id};
5345                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5346             }
5347         }
5348
5349         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5350     };
5351     warn $@ if $@; # avoid errors - just warn
5352 }
5353
5354 # call only in locked context
5355 sub _do_vm_stop {
5356     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5357
5358     my $pid = check_running($vmid, $nocheck);
5359     return if !$pid;
5360
5361     my $conf;
5362     if (!$nocheck) {
5363         $conf = PVE::QemuConfig->load_config($vmid);
5364         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5365         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5366             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5367             $timeout = $opts->{down} if $opts->{down};
5368         }
5369         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5370     }
5371
5372     eval {
5373         if ($shutdown) {
5374             if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5375                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5376             } else {
5377                 mon_cmd($vmid, "system_powerdown");
5378             }
5379         } else {
5380             mon_cmd($vmid, "quit");
5381         }
5382     };
5383     my $err = $@;
5384
5385     if (!$err) {
5386         $timeout = 60 if !defined($timeout);
5387
5388         my $count = 0;
5389         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5390             $count++;
5391             sleep 1;
5392         }
5393
5394         if ($count >= $timeout) {
5395             if ($force) {
5396                 warn "VM still running - terminating now with SIGTERM\n";
5397                 kill 15, $pid;
5398             } else {
5399                 die "VM quit/powerdown failed - got timeout\n";
5400             }
5401         } else {
5402             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5403             return;
5404         }
5405     } else {
5406        if (!check_running($vmid, $nocheck)) {
5407             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5408             return;
5409        }
5410        if ($force) {
5411             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5412             kill 15, $pid;
5413         } else {
5414             die "VM quit/powerdown failed\n";
5415         }
5416     }
5417
5418     # wait again
5419     $timeout = 10;
5420
5421     my $count = 0;
5422     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5423         $count++;
5424         sleep 1;
5425     }
5426
5427     if ($count >= $timeout) {
5428         warn "VM still running - terminating now with SIGKILL\n";
5429         kill 9, $pid;
5430         sleep 1;
5431     }
5432
5433     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5434 }
5435
5436 # Note: use $nocheck to skip tests if VM configuration file exists.
5437 # We need that when migration VMs to other nodes (files already moved)
5438 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5439 sub vm_stop {
5440     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5441
5442     $force = 1 if !defined($force) && !$shutdown;
5443
5444     if ($migratedfrom){
5445         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5446         kill 15, $pid if $pid;
5447         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5448         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5449         return;
5450     }
5451
5452     PVE::QemuConfig->lock_config($vmid, sub {
5453         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5454    });
5455 }
5456
5457 sub vm_reboot {
5458     my ($vmid, $timeout) = @_;
5459
5460     PVE::QemuConfig->lock_config($vmid, sub {
5461         eval {
5462
5463             # only reboot if running, as qmeventd starts it again on a stop event
5464             return if !check_running($vmid);
5465
5466             create_reboot_request($vmid);
5467
5468             my $storecfg = PVE::Storage::config();
5469             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5470
5471         };
5472         if (my $err = $@) {
5473             # avoid that the next normal shutdown will be confused for a reboot
5474             clear_reboot_request($vmid);
5475             die $err;
5476         }
5477    });
5478 }
5479
5480 # note: if using the statestorage parameter, the caller has to check privileges
5481 sub vm_suspend {
5482     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5483
5484     my $conf;
5485     my $path;
5486     my $storecfg;
5487     my $vmstate;
5488
5489     PVE::QemuConfig->lock_config($vmid, sub {
5490
5491         $conf = PVE::QemuConfig->load_config($vmid);
5492
5493         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5494         PVE::QemuConfig->check_lock($conf)
5495             if !($skiplock || $is_backing_up);
5496
5497         die "cannot suspend to disk during backup\n"
5498             if $is_backing_up && $includestate;
5499
5500         if ($includestate) {
5501             $conf->{lock} = 'suspending';
5502             my $date = strftime("%Y-%m-%d", localtime(time()));
5503             $storecfg = PVE::Storage::config();
5504             if (!$statestorage) {
5505                 $statestorage = find_vmstate_storage($conf, $storecfg);
5506                 # check permissions for the storage
5507                 my $rpcenv = PVE::RPCEnvironment::get();
5508                 if ($rpcenv->{type} ne 'cli') {
5509                     my $authuser = $rpcenv->get_user();
5510                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5511                 }
5512             }
5513
5514
5515             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
5516                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5517             $path = PVE::Storage::path($storecfg, $vmstate);
5518             PVE::QemuConfig->write_config($vmid, $conf);
5519         } else {
5520             mon_cmd($vmid, "stop");
5521         }
5522     });
5523
5524     if ($includestate) {
5525         # save vm state
5526         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5527
5528         eval {
5529             mon_cmd($vmid, "savevm-start", statefile => $path);
5530             for(;;) {
5531                 my $state = mon_cmd($vmid, "query-savevm");
5532                 if (!$state->{status}) {
5533                     die "savevm not active\n";
5534                 } elsif ($state->{status} eq 'active') {
5535                     sleep(1);
5536                     next;
5537                 } elsif ($state->{status} eq 'completed') {
5538                     print "State saved, quitting\n";
5539                     last;
5540                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5541                     die "query-savevm failed with error '$state->{error}'\n"
5542                 } else {
5543                     die "query-savevm returned status '$state->{status}'\n";
5544                 }
5545             }
5546         };
5547         my $err = $@;
5548
5549         PVE::QemuConfig->lock_config($vmid, sub {
5550             $conf = PVE::QemuConfig->load_config($vmid);
5551             if ($err) {
5552                 # cleanup, but leave suspending lock, to indicate something went wrong
5553                 eval {
5554                     mon_cmd($vmid, "savevm-end");
5555                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5556                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5557                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
5558                     PVE::QemuConfig->write_config($vmid, $conf);
5559                 };
5560                 warn $@ if $@;
5561                 die $err;
5562             }
5563
5564             die "lock changed unexpectedly\n"
5565                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5566
5567             mon_cmd($vmid, "quit");
5568             $conf->{lock} = 'suspended';
5569             PVE::QemuConfig->write_config($vmid, $conf);
5570         });
5571     }
5572 }
5573
5574 sub vm_resume {
5575     my ($vmid, $skiplock, $nocheck) = @_;
5576
5577     PVE::QemuConfig->lock_config($vmid, sub {
5578         my $res = mon_cmd($vmid, 'query-status');
5579         my $resume_cmd = 'cont';
5580
5581         if ($res->{status} && $res->{status} eq 'suspended') {
5582             $resume_cmd = 'system_wakeup';
5583         }
5584
5585         if (!$nocheck) {
5586
5587             my $conf = PVE::QemuConfig->load_config($vmid);
5588
5589             PVE::QemuConfig->check_lock($conf)
5590                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5591         }
5592
5593         mon_cmd($vmid, $resume_cmd);
5594     });
5595 }
5596
5597 sub vm_sendkey {
5598     my ($vmid, $skiplock, $key) = @_;
5599
5600     PVE::QemuConfig->lock_config($vmid, sub {
5601
5602         my $conf = PVE::QemuConfig->load_config($vmid);
5603
5604         # there is no qmp command, so we use the human monitor command
5605         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5606         die $res if $res ne '';
5607     });
5608 }
5609
5610 # vzdump restore implementaion
5611
5612 sub tar_archive_read_firstfile {
5613     my $archive = shift;
5614
5615     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5616
5617     # try to detect archive type first
5618     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5619         die "unable to open file '$archive'\n";
5620     my $firstfile = <$fh>;
5621     kill 15, $pid;
5622     close $fh;
5623
5624     die "ERROR: archive contaions no data\n" if !$firstfile;
5625     chomp $firstfile;
5626
5627     return $firstfile;
5628 }
5629
5630 sub tar_restore_cleanup {
5631     my ($storecfg, $statfile) = @_;
5632
5633     print STDERR "starting cleanup\n";
5634
5635     if (my $fd = IO::File->new($statfile, "r")) {
5636         while (defined(my $line = <$fd>)) {
5637             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5638                 my $volid = $2;
5639                 eval {
5640                     if ($volid =~ m|^/|) {
5641                         unlink $volid || die 'unlink failed\n';
5642                     } else {
5643                         PVE::Storage::vdisk_free($storecfg, $volid);
5644                     }
5645                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5646                 };
5647                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5648             } else {
5649                 print STDERR "unable to parse line in statfile - $line";
5650             }
5651         }
5652         $fd->close();
5653     }
5654 }
5655
5656 sub restore_file_archive {
5657     my ($archive, $vmid, $user, $opts) = @_;
5658
5659     return restore_vma_archive($archive, $vmid, $user, $opts)
5660         if $archive eq '-';
5661
5662     my $info = PVE::Storage::archive_info($archive);
5663     my $format = $opts->{format} // $info->{format};
5664     my $comp = $info->{compression};
5665
5666     # try to detect archive format
5667     if ($format eq 'tar') {
5668         return restore_tar_archive($archive, $vmid, $user, $opts);
5669     } else {
5670         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5671     }
5672 }
5673
5674 # hepler to remove disks that will not be used after restore
5675 my $restore_cleanup_oldconf = sub {
5676     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
5677
5678     PVE::QemuConfig->foreach_volume($oldconf, sub {
5679         my ($ds, $drive) = @_;
5680
5681         return if drive_is_cdrom($drive, 1);
5682
5683         my $volid = $drive->{file};
5684         return if !$volid || $volid =~ m|^/|;
5685
5686         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
5687         return if !$path || !$owner || ($owner != $vmid);
5688
5689         # Note: only delete disk we want to restore
5690         # other volumes will become unused
5691         if ($virtdev_hash->{$ds}) {
5692             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
5693             if (my $err = $@) {
5694                 warn $err;
5695             }
5696         }
5697     });
5698
5699     # delete vmstate files, after the restore we have no snapshots anymore
5700     foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5701         my $snap = $oldconf->{snapshots}->{$snapname};
5702         if ($snap->{vmstate}) {
5703             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
5704             if (my $err = $@) {
5705                 warn $err;
5706             }
5707         }
5708     }
5709 };
5710
5711 # Helper to parse vzdump backup device hints
5712 #
5713 # $rpcenv: Environment, used to ckeck storage permissions
5714 # $user: User ID, to check storage permissions
5715 # $storecfg: Storage configuration
5716 # $fh: the file handle for reading the configuration
5717 # $devinfo: should contain device sizes for all backu-up'ed devices
5718 # $options: backup options (pool, default storage)
5719 #
5720 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
5721 my $parse_backup_hints = sub {
5722     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
5723
5724     my $virtdev_hash = {};
5725
5726     while (defined(my $line = <$fh>)) {
5727         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5728             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5729             die "archive does not contain data for drive '$virtdev'\n"
5730                 if !$devinfo->{$devname};
5731
5732             if (defined($options->{storage})) {
5733                 $storeid = $options->{storage} || 'local';
5734             } elsif (!$storeid) {
5735                 $storeid = 'local';
5736             }
5737             $format = 'raw' if !$format;
5738             $devinfo->{$devname}->{devname} = $devname;
5739             $devinfo->{$devname}->{virtdev} = $virtdev;
5740             $devinfo->{$devname}->{format} = $format;
5741             $devinfo->{$devname}->{storeid} = $storeid;
5742
5743             # check permission on storage
5744             my $pool = $options->{pool}; # todo: do we need that?
5745             if ($user ne 'root@pam') {
5746                 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5747             }
5748
5749             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5750         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
5751             my $virtdev = $1;
5752             my $drive = parse_drive($virtdev, $2);
5753             if (drive_is_cloudinit($drive)) {
5754                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5755                 $storeid = $options->{storage} if defined ($options->{storage});
5756                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5757                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
5758
5759                 $virtdev_hash->{$virtdev} = {
5760                     format => $format,
5761                     storeid => $storeid,
5762                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
5763                     is_cloudinit => 1,
5764                 };
5765             }
5766         }
5767     }
5768
5769     return $virtdev_hash;
5770 };
5771
5772 # Helper to allocate and activate all volumes required for a restore
5773 #
5774 # $storecfg: Storage configuration
5775 # $virtdev_hash: as returned by parse_backup_hints()
5776 #
5777 # Returns: { $virtdev => $volid }
5778 my $restore_allocate_devices = sub {
5779     my ($storecfg, $virtdev_hash, $vmid) = @_;
5780
5781     my $map = {};
5782     foreach my $virtdev (sort keys %$virtdev_hash) {
5783         my $d = $virtdev_hash->{$virtdev};
5784         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5785         my $storeid = $d->{storeid};
5786         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5787
5788         # test if requested format is supported
5789         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5790         my $supported = grep { $_ eq $d->{format} } @$validFormats;
5791         $d->{format} = $defFormat if !$supported;
5792
5793         my $name;
5794         if ($d->{is_cloudinit}) {
5795             $name = "vm-$vmid-cloudinit";
5796             $name .= ".$d->{format}" if $d->{format} ne 'raw';
5797         }
5798
5799         my $volid = PVE::Storage::vdisk_alloc(
5800             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
5801
5802         print STDERR "new volume ID is '$volid'\n";
5803         $d->{volid} = $volid;
5804
5805         PVE::Storage::activate_volumes($storecfg, [$volid]);
5806
5807         $map->{$virtdev} = $volid;
5808     }
5809
5810     return $map;
5811 };
5812
5813 my $restore_update_config_line = sub {
5814     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5815
5816     return if $line =~ m/^\#qmdump\#/;
5817     return if $line =~ m/^\#vzdump\#/;
5818     return if $line =~ m/^lock:/;
5819     return if $line =~ m/^unused\d+:/;
5820     return if $line =~ m/^parent:/;
5821
5822     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5823     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5824         # try to convert old 1.X settings
5825         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5826         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5827             my ($model, $macaddr) = split(/\=/, $devconfig);
5828             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5829             my $net = {
5830                 model => $model,
5831                 bridge => "vmbr$ind",
5832                 macaddr => $macaddr,
5833             };
5834             my $netstr = print_net($net);
5835
5836             print $outfd "net$cookie->{netcount}: $netstr\n";
5837             $cookie->{netcount}++;
5838         }
5839     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5840         my ($id, $netstr) = ($1, $2);
5841         my $net = parse_net($netstr);
5842         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5843         $netstr = print_net($net);
5844         print $outfd "$id: $netstr\n";
5845     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5846         my $virtdev = $1;
5847         my $value = $3;
5848         my $di = parse_drive($virtdev, $value);
5849         if (defined($di->{backup}) && !$di->{backup}) {
5850             print $outfd "#$line";
5851         } elsif ($map->{$virtdev}) {
5852             delete $di->{format}; # format can change on restore
5853             $di->{file} = $map->{$virtdev};
5854             $value = print_drive($di);
5855             print $outfd "$virtdev: $value\n";
5856         } else {
5857             print $outfd $line;
5858         }
5859     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5860         my $vmgenid = $1;
5861         if ($vmgenid ne '0') {
5862             # always generate a new vmgenid if there was a valid one setup
5863             $vmgenid = generate_uuid();
5864         }
5865         print $outfd "vmgenid: $vmgenid\n";
5866     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5867         my ($uuid, $uuid_str);
5868         UUID::generate($uuid);
5869         UUID::unparse($uuid, $uuid_str);
5870         my $smbios1 = parse_smbios1($2);
5871         $smbios1->{uuid} = $uuid_str;
5872         print $outfd $1.print_smbios1($smbios1)."\n";
5873     } else {
5874         print $outfd $line;
5875     }
5876 };
5877
5878 my $restore_deactivate_volumes = sub {
5879     my ($storecfg, $devinfo) = @_;
5880
5881     my $vollist = [];
5882     foreach my $devname (keys %$devinfo) {
5883         my $volid = $devinfo->{$devname}->{volid};
5884         push @$vollist, $volid if $volid;
5885     }
5886
5887     PVE::Storage::deactivate_volumes($storecfg, $vollist);
5888 };
5889
5890 my $restore_destroy_volumes = sub {
5891     my ($storecfg, $devinfo) = @_;
5892
5893     foreach my $devname (keys %$devinfo) {
5894         my $volid = $devinfo->{$devname}->{volid};
5895         next if !$volid;
5896         eval {
5897             if ($volid =~ m|^/|) {
5898                 unlink $volid || die 'unlink failed\n';
5899             } else {
5900                 PVE::Storage::vdisk_free($storecfg, $volid);
5901             }
5902             print STDERR "temporary volume '$volid' sucessfuly removed\n";
5903         };
5904         print STDERR "unable to cleanup '$volid' - $@" if $@;
5905     }
5906 };
5907
5908 sub scan_volids {
5909     my ($cfg, $vmid) = @_;
5910
5911     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5912
5913     my $volid_hash = {};
5914     foreach my $storeid (keys %$info) {
5915         foreach my $item (@{$info->{$storeid}}) {
5916             next if !($item->{volid} && $item->{size});
5917             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5918             $volid_hash->{$item->{volid}} = $item;
5919         }
5920     }
5921
5922     return $volid_hash;
5923 }
5924
5925 sub update_disk_config {
5926     my ($vmid, $conf, $volid_hash) = @_;
5927
5928     my $changes;
5929     my $prefix = "VM $vmid";
5930
5931     # used and unused disks
5932     my $referenced = {};
5933
5934     # Note: it is allowed to define multiple storages with same path (alias), so
5935     # we need to check both 'volid' and real 'path' (two different volid can point
5936     # to the same path).
5937
5938     my $referencedpath = {};
5939
5940     # update size info
5941     PVE::QemuConfig->foreach_volume($conf, sub {
5942         my ($opt, $drive) = @_;
5943
5944         my $volid = $drive->{file};
5945         return if !$volid;
5946         my $volume = $volid_hash->{$volid};
5947
5948         # mark volid as "in-use" for next step
5949         $referenced->{$volid} = 1;
5950         if ($volume && (my $path = $volume->{path})) {
5951             $referencedpath->{$path} = 1;
5952         }
5953
5954         return if drive_is_cdrom($drive);
5955         return if !$volume;
5956
5957         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
5958         if (defined($updated)) {
5959             $changes = 1;
5960             $conf->{$opt} = print_drive($updated);
5961             print "$prefix ($opt): $msg\n";
5962         }
5963     });
5964
5965     # remove 'unusedX' entry if volume is used
5966     PVE::QemuConfig->foreach_unused_volume($conf, sub {
5967         my ($opt, $drive) = @_;
5968
5969         my $volid = $drive->{file};
5970         return if !$volid;
5971
5972         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5973         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5974             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
5975             $changes = 1;
5976             delete $conf->{$opt};
5977         }
5978
5979         $referenced->{$volid} = 1;
5980         $referencedpath->{$path} = 1 if $path;
5981     });
5982
5983     foreach my $volid (sort keys %$volid_hash) {
5984         next if $volid =~ m/vm-$vmid-state-/;
5985         next if $referenced->{$volid};
5986         my $path = $volid_hash->{$volid}->{path};
5987         next if !$path; # just to be sure
5988         next if $referencedpath->{$path};
5989         $changes = 1;
5990         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
5991         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
5992         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5993     }
5994
5995     return $changes;
5996 }
5997
5998 sub rescan {
5999     my ($vmid, $nolock, $dryrun) = @_;
6000
6001     my $cfg = PVE::Storage::config();
6002
6003     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6004     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6005     foreach my $stor (keys %{$cfg->{ids}}) {
6006         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6007     }
6008
6009     print "rescan volumes...\n";
6010     my $volid_hash = scan_volids($cfg, $vmid);
6011
6012     my $updatefn =  sub {
6013         my ($vmid) = @_;
6014
6015         my $conf = PVE::QemuConfig->load_config($vmid);
6016
6017         PVE::QemuConfig->check_lock($conf);
6018
6019         my $vm_volids = {};
6020         foreach my $volid (keys %$volid_hash) {
6021             my $info = $volid_hash->{$volid};
6022             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6023         }
6024
6025         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6026
6027         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6028     };
6029
6030     if (defined($vmid)) {
6031         if ($nolock) {
6032             &$updatefn($vmid);
6033         } else {
6034             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6035         }
6036     } else {
6037         my $vmlist = config_list();
6038         foreach my $vmid (keys %$vmlist) {
6039             if ($nolock) {
6040                 &$updatefn($vmid);
6041             } else {
6042                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6043             }
6044         }
6045     }
6046 }
6047
6048 sub restore_proxmox_backup_archive {
6049     my ($archive, $vmid, $user, $options) = @_;
6050
6051     my $storecfg = PVE::Storage::config();
6052
6053     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6054     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6055
6056     my $server = $scfg->{server};
6057     my $datastore = $scfg->{datastore};
6058     my $username = $scfg->{username} // 'root@pam';
6059     my $fingerprint = $scfg->{fingerprint};
6060     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6061
6062     my $repo = "$username\@$server:$datastore";
6063
6064     # This is only used for `pbs-restore`!
6065     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6066     local $ENV{PBS_PASSWORD} = $password;
6067     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6068
6069     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6070         PVE::Storage::parse_volname($storecfg, $archive);
6071
6072     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6073
6074     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6075
6076     my $tmpdir = "/var/tmp/vzdumptmp$$";
6077     rmtree $tmpdir;
6078     mkpath $tmpdir;
6079
6080     my $conffile = PVE::QemuConfig->config_file($vmid);
6081     my $tmpfn = "$conffile.$$.tmp";
6082      # disable interrupts (always do cleanups)
6083     local $SIG{INT} =
6084         local $SIG{TERM} =
6085         local $SIG{QUIT} =
6086         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6087
6088     # Note: $oldconf is undef if VM does not exists
6089     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6090     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6091
6092     my $rpcenv = PVE::RPCEnvironment::get();
6093     my $devinfo = {};
6094
6095     eval {
6096         # enable interrupts
6097         local $SIG{INT} =
6098             local $SIG{TERM} =
6099             local $SIG{QUIT} =
6100             local $SIG{HUP} =
6101             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6102
6103         my $cfgfn = "$tmpdir/qemu-server.conf";
6104         my $firewall_config_fn = "$tmpdir/fw.conf";
6105         my $index_fn = "$tmpdir/index.json";
6106
6107         my $cmd = "restore";
6108
6109         my $param = [$pbs_backup_name, "index.json", $index_fn];
6110         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6111         my $index = PVE::Tools::file_get_contents($index_fn);
6112         $index = decode_json($index);
6113
6114         # print Dumper($index);
6115         foreach my $info (@{$index->{files}}) {
6116             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6117                 my $devname = $1;
6118                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6119                     $devinfo->{$devname}->{size} = $1;
6120                 } else {
6121                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6122                 }
6123             }
6124         }
6125
6126         my $is_qemu_server_backup = scalar(
6127             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6128         );
6129         if (!$is_qemu_server_backup) {
6130             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6131         }
6132         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6133
6134         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6135         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6136
6137         if ($has_firewall_config) {
6138             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6139             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6140
6141             my $pve_firewall_dir = '/etc/pve/firewall';
6142             mkdir $pve_firewall_dir; # make sure the dir exists
6143             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6144         }
6145
6146         my $fh = IO::File->new($cfgfn, "r") ||
6147             die "unable to read qemu-server.conf - $!\n";
6148
6149         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6150
6151         # fixme: rate limit?
6152
6153         # create empty/temp config
6154         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6155
6156         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6157
6158         # allocate volumes
6159         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6160
6161         foreach my $virtdev (sort keys %$virtdev_hash) {
6162             my $d = $virtdev_hash->{$virtdev};
6163             next if $d->{is_cloudinit}; # no need to restore cloudinit
6164
6165             my $volid = $d->{volid};
6166
6167             my $path = PVE::Storage::path($storecfg, $volid);
6168
6169             # This is the ONLY user of the PBS_ env vars set on top of this function!
6170             my $pbs_restore_cmd = [
6171                 '/usr/bin/pbs-restore',
6172                 '--repository', $repo,
6173                 $pbs_backup_name,
6174                 "$d->{devname}.img.fidx",
6175                 $path,
6176                 '--verbose',
6177                 ];
6178
6179             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6180             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
6181
6182             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6183                 push @$pbs_restore_cmd, '--skip-zero';
6184             }
6185
6186             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6187             print "restore proxmox backup image: $dbg_cmdstring\n";
6188             run_command($pbs_restore_cmd);
6189         }
6190
6191         $fh->seek(0, 0) || die "seek failed - $!\n";
6192
6193         my $outfd = new IO::File ($tmpfn, "w") ||
6194             die "unable to write config for VM $vmid\n";
6195
6196         my $cookie = { netcount => 0 };
6197         while (defined(my $line = <$fh>)) {
6198             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $options->{unique});
6199         }
6200
6201         $fh->close();
6202         $outfd->close();
6203     };
6204     my $err = $@;
6205
6206     $restore_deactivate_volumes->($storecfg, $devinfo);
6207
6208     rmtree $tmpdir;
6209
6210     if ($err) {
6211         unlink $tmpfn;
6212         $restore_destroy_volumes->($storecfg, $devinfo);
6213         die $err;
6214     }
6215
6216     rename($tmpfn, $conffile) ||
6217         die "unable to commit configuration file '$conffile'\n";
6218
6219     PVE::Cluster::cfs_update(); # make sure we read new file
6220
6221     eval { rescan($vmid, 1); };
6222     warn $@ if $@;
6223 }
6224
6225 sub restore_vma_archive {
6226     my ($archive, $vmid, $user, $opts, $comp) = @_;
6227
6228     my $readfrom = $archive;
6229
6230     my $cfg = PVE::Storage::config();
6231     my $commands = [];
6232     my $bwlimit = $opts->{bwlimit};
6233
6234     my $dbg_cmdstring = '';
6235     my $add_pipe = sub {
6236         my ($cmd) = @_;
6237         push @$commands, $cmd;
6238         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6239         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6240         $readfrom = '-';
6241     };
6242
6243     my $input = undef;
6244     if ($archive eq '-') {
6245         $input = '<&STDIN';
6246     } else {
6247         # If we use a backup from a PVE defined storage we also consider that
6248         # storage's rate limit:
6249         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6250         if (defined($volid)) {
6251             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6252             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6253             if ($readlimit) {
6254                 print STDERR "applying read rate limit: $readlimit\n";
6255                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6256                 $add_pipe->($cstream);
6257             }
6258         }
6259     }
6260
6261     if ($comp) {
6262         my $info = PVE::Storage::decompressor_info('vma', $comp);
6263         my $cmd = $info->{decompressor};
6264         push @$cmd, $readfrom;
6265         $add_pipe->($cmd);
6266     }
6267
6268     my $tmpdir = "/var/tmp/vzdumptmp$$";
6269     rmtree $tmpdir;
6270
6271     # disable interrupts (always do cleanups)
6272     local $SIG{INT} =
6273         local $SIG{TERM} =
6274         local $SIG{QUIT} =
6275         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6276
6277     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6278     POSIX::mkfifo($mapfifo, 0600);
6279     my $fifofh;
6280
6281     my $openfifo = sub {
6282         open($fifofh, '>', $mapfifo) || die $!;
6283     };
6284
6285     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6286
6287     my $oldtimeout;
6288     my $timeout = 5;
6289
6290     my $devinfo = {};
6291
6292     my $rpcenv = PVE::RPCEnvironment::get();
6293
6294     my $conffile = PVE::QemuConfig->config_file($vmid);
6295     my $tmpfn = "$conffile.$$.tmp";
6296
6297     # Note: $oldconf is undef if VM does not exist
6298     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6299     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6300
6301     my %storage_limits;
6302
6303     my $print_devmap = sub {
6304         my $cfgfn = "$tmpdir/qemu-server.conf";
6305
6306         # we can read the config - that is already extracted
6307         my $fh = IO::File->new($cfgfn, "r") ||
6308             die "unable to read qemu-server.conf - $!\n";
6309
6310         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6311         if (-f $fwcfgfn) {
6312             my $pve_firewall_dir = '/etc/pve/firewall';
6313             mkdir $pve_firewall_dir; # make sure the dir exists
6314             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6315         }
6316
6317         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6318
6319         foreach my $key (keys %storage_limits) {
6320             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6321             next if !$limit;
6322             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6323             $storage_limits{$key} = $limit * 1024;
6324         }
6325
6326         foreach my $devname (keys %$devinfo) {
6327             die "found no device mapping information for device '$devname'\n"
6328                 if !$devinfo->{$devname}->{virtdev};
6329         }
6330
6331         # create empty/temp config
6332         if ($oldconf) {
6333             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6334             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6335         }
6336
6337         # allocate volumes
6338         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6339
6340         # print restore information to $fifofh
6341         foreach my $virtdev (sort keys %$virtdev_hash) {
6342             my $d = $virtdev_hash->{$virtdev};
6343             next if $d->{is_cloudinit}; # no need to restore cloudinit
6344
6345             my $storeid = $d->{storeid};
6346             my $volid = $d->{volid};
6347
6348             my $map_opts = '';
6349             if (my $limit = $storage_limits{$storeid}) {
6350                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6351             }
6352
6353             my $write_zeros = 1;
6354             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6355                 $write_zeros = 0;
6356             }
6357
6358             my $path = PVE::Storage::path($cfg, $volid);
6359
6360             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6361
6362             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6363         }
6364
6365         $fh->seek(0, 0) || die "seek failed - $!\n";
6366
6367         my $outfd = new IO::File ($tmpfn, "w") ||
6368             die "unable to write config for VM $vmid\n";
6369
6370         my $cookie = { netcount => 0 };
6371         while (defined(my $line = <$fh>)) {
6372             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6373         }
6374
6375         $fh->close();
6376         $outfd->close();
6377     };
6378
6379     eval {
6380         # enable interrupts
6381         local $SIG{INT} =
6382             local $SIG{TERM} =
6383             local $SIG{QUIT} =
6384             local $SIG{HUP} =
6385             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6386         local $SIG{ALRM} = sub { die "got timeout\n"; };
6387
6388         $oldtimeout = alarm($timeout);
6389
6390         my $parser = sub {
6391             my $line = shift;
6392
6393             print "$line\n";
6394
6395             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6396                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6397                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6398             } elsif ($line =~ m/^CTIME: /) {
6399                 # we correctly received the vma config, so we can disable
6400                 # the timeout now for disk allocation (set to 10 minutes, so
6401                 # that we always timeout if something goes wrong)
6402                 alarm(600);
6403                 &$print_devmap();
6404                 print $fifofh "done\n";
6405                 my $tmp = $oldtimeout || 0;
6406                 $oldtimeout = undef;
6407                 alarm($tmp);
6408                 close($fifofh);
6409             }
6410         };
6411
6412         print "restore vma archive: $dbg_cmdstring\n";
6413         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6414     };
6415     my $err = $@;
6416
6417     alarm($oldtimeout) if $oldtimeout;
6418
6419     $restore_deactivate_volumes->($cfg, $devinfo);
6420
6421     unlink $mapfifo;
6422     rmtree $tmpdir;
6423
6424     if ($err) {
6425         unlink $tmpfn;
6426         $restore_destroy_volumes->($cfg, $devinfo);
6427         die $err;
6428     }
6429
6430     rename($tmpfn, $conffile) ||
6431         die "unable to commit configuration file '$conffile'\n";
6432
6433     PVE::Cluster::cfs_update(); # make sure we read new file
6434
6435     eval { rescan($vmid, 1); };
6436     warn $@ if $@;
6437 }
6438
6439 sub restore_tar_archive {
6440     my ($archive, $vmid, $user, $opts) = @_;
6441
6442     if ($archive ne '-') {
6443         my $firstfile = tar_archive_read_firstfile($archive);
6444         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
6445             if $firstfile ne 'qemu-server.conf';
6446     }
6447
6448     my $storecfg = PVE::Storage::config();
6449
6450     # avoid zombie disks when restoring over an existing VM -> cleanup first
6451     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6452     # skiplock=1 because qmrestore has set the 'create' lock itself already
6453     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6454     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6455
6456     my $tocmd = "/usr/lib/qemu-server/qmextract";
6457
6458     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6459     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6460     $tocmd .= ' --prealloc' if $opts->{prealloc};
6461     $tocmd .= ' --info' if $opts->{info};
6462
6463     # tar option "xf" does not autodetect compression when read from STDIN,
6464     # so we pipe to zcat
6465     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6466         PVE::Tools::shellquote("--to-command=$tocmd");
6467
6468     my $tmpdir = "/var/tmp/vzdumptmp$$";
6469     mkpath $tmpdir;
6470
6471     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6472     local $ENV{VZDUMP_VMID} = $vmid;
6473     local $ENV{VZDUMP_USER} = $user;
6474
6475     my $conffile = PVE::QemuConfig->config_file($vmid);
6476     my $tmpfn = "$conffile.$$.tmp";
6477
6478     # disable interrupts (always do cleanups)
6479     local $SIG{INT} =
6480         local $SIG{TERM} =
6481         local $SIG{QUIT} =
6482         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6483
6484     eval {
6485         # enable interrupts
6486         local $SIG{INT} =
6487             local $SIG{TERM} =
6488             local $SIG{QUIT} =
6489             local $SIG{HUP} =
6490             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6491
6492         if ($archive eq '-') {
6493             print "extracting archive from STDIN\n";
6494             run_command($cmd, input => "<&STDIN");
6495         } else {
6496             print "extracting archive '$archive'\n";
6497             run_command($cmd);
6498         }
6499
6500         return if $opts->{info};
6501
6502         # read new mapping
6503         my $map = {};
6504         my $statfile = "$tmpdir/qmrestore.stat";
6505         if (my $fd = IO::File->new($statfile, "r")) {
6506             while (defined (my $line = <$fd>)) {
6507                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6508                     $map->{$1} = $2 if $1;
6509                 } else {
6510                     print STDERR "unable to parse line in statfile - $line\n";
6511                 }
6512             }
6513             $fd->close();
6514         }
6515
6516         my $confsrc = "$tmpdir/qemu-server.conf";
6517
6518         my $srcfd = new IO::File($confsrc, "r") ||
6519             die "unable to open file '$confsrc'\n";
6520
6521         my $outfd = new IO::File ($tmpfn, "w") ||
6522             die "unable to write config for VM $vmid\n";
6523
6524         my $cookie = { netcount => 0 };
6525         while (defined (my $line = <$srcfd>)) {
6526             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6527         }
6528
6529         $srcfd->close();
6530         $outfd->close();
6531     };
6532     if (my $err = $@) {
6533         unlink $tmpfn;
6534         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6535         die $err;
6536     }
6537
6538     rmtree $tmpdir;
6539
6540     rename $tmpfn, $conffile ||
6541         die "unable to commit configuration file '$conffile'\n";
6542
6543     PVE::Cluster::cfs_update(); # make sure we read new file
6544
6545     eval { rescan($vmid, 1); };
6546     warn $@ if $@;
6547 };
6548
6549 sub foreach_storage_used_by_vm {
6550     my ($conf, $func) = @_;
6551
6552     my $sidhash = {};
6553
6554     PVE::QemuConfig->foreach_volume($conf, sub {
6555         my ($ds, $drive) = @_;
6556         return if drive_is_cdrom($drive);
6557
6558         my $volid = $drive->{file};
6559
6560         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6561         $sidhash->{$sid} = $sid if $sid;
6562     });
6563
6564     foreach my $sid (sort keys %$sidhash) {
6565         &$func($sid);
6566     }
6567 }
6568
6569 my $qemu_snap_storage = {
6570     rbd => 1,
6571 };
6572 sub do_snapshots_with_qemu {
6573     my ($storecfg, $volid) = @_;
6574
6575     my $storage_name = PVE::Storage::parse_volume_id($volid);
6576     my $scfg = $storecfg->{ids}->{$storage_name};
6577
6578     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6579         return 1;
6580     }
6581
6582     if ($volid =~ m/\.(qcow2|qed)$/){
6583         return 1;
6584     }
6585
6586     return;
6587 }
6588
6589 sub qga_check_running {
6590     my ($vmid, $nowarn) = @_;
6591
6592     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6593     if ($@) {
6594         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6595         return 0;
6596     }
6597     return 1;
6598 }
6599
6600 sub template_create {
6601     my ($vmid, $conf, $disk) = @_;
6602
6603     my $storecfg = PVE::Storage::config();
6604
6605     PVE::QemuConfig->foreach_volume($conf, sub {
6606         my ($ds, $drive) = @_;
6607
6608         return if drive_is_cdrom($drive);
6609         return if $disk && $ds ne $disk;
6610
6611         my $volid = $drive->{file};
6612         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6613
6614         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6615         $drive->{file} = $voliddst;
6616         $conf->{$ds} = print_drive($drive);
6617         PVE::QemuConfig->write_config($vmid, $conf);
6618     });
6619 }
6620
6621 sub convert_iscsi_path {
6622     my ($path) = @_;
6623
6624     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6625         my $portal = $1;
6626         my $target = $2;
6627         my $lun = $3;
6628
6629         my $initiator_name = get_initiator_name();
6630
6631         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6632                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6633     }
6634
6635     die "cannot convert iscsi path '$path', unkown format\n";
6636 }
6637
6638 sub qemu_img_convert {
6639     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6640
6641     my $storecfg = PVE::Storage::config();
6642     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6643     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6644
6645     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6646
6647     my $cachemode;
6648     my $src_path;
6649     my $src_is_iscsi = 0;
6650     my $src_format;
6651
6652     if ($src_storeid) {
6653         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6654         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6655         $src_format = qemu_img_format($src_scfg, $src_volname);
6656         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6657         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6658         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6659     } elsif (-f $src_volid) {
6660         $src_path = $src_volid;
6661         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6662             $src_format = $1;
6663         }
6664     }
6665
6666     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6667
6668     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6669     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6670     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6671     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6672
6673     my $cmd = [];
6674     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6675     push @$cmd, '-l', "snapshot.name=$snapname"
6676         if $snapname && $src_format && $src_format eq "qcow2";
6677     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6678     push @$cmd, '-T', $cachemode if defined($cachemode);
6679
6680     if ($src_is_iscsi) {
6681         push @$cmd, '--image-opts';
6682         $src_path = convert_iscsi_path($src_path);
6683     } elsif ($src_format) {
6684         push @$cmd, '-f', $src_format;
6685     }
6686
6687     if ($dst_is_iscsi) {
6688         push @$cmd, '--target-image-opts';
6689         $dst_path = convert_iscsi_path($dst_path);
6690     } else {
6691         push @$cmd, '-O', $dst_format;
6692     }
6693
6694     push @$cmd, $src_path;
6695
6696     if (!$dst_is_iscsi && $is_zero_initialized) {
6697         push @$cmd, "zeroinit:$dst_path";
6698     } else {
6699         push @$cmd, $dst_path;
6700     }
6701
6702     my $parser = sub {
6703         my $line = shift;
6704         if($line =~ m/\((\S+)\/100\%\)/){
6705             my $percent = $1;
6706             my $transferred = int($size * $percent / 100);
6707             my $remaining = $size - $transferred;
6708
6709             print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6710         }
6711
6712     };
6713
6714     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6715     my $err = $@;
6716     die "copy failed: $err" if $err;
6717 }
6718
6719 sub qemu_img_format {
6720     my ($scfg, $volname) = @_;
6721
6722     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6723         return $1;
6724     } else {
6725         return "raw";
6726     }
6727 }
6728
6729 sub qemu_drive_mirror {
6730     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
6731
6732     $jobs = {} if !$jobs;
6733
6734     my $qemu_target;
6735     my $format;
6736     $jobs->{"drive-$drive"} = {};
6737
6738     if ($dst_volid =~ /^nbd:/) {
6739         $qemu_target = $dst_volid;
6740         $format = "nbd";
6741     } else {
6742         my $storecfg = PVE::Storage::config();
6743         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6744
6745         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6746
6747         $format = qemu_img_format($dst_scfg, $dst_volname);
6748
6749         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6750
6751         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6752     }
6753
6754     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6755     $opts->{format} = $format if $format;
6756
6757     if (defined($src_bitmap)) {
6758         $opts->{sync} = 'incremental';
6759         $opts->{bitmap} = $src_bitmap;
6760         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
6761     }
6762
6763     if (defined($bwlimit)) {
6764         $opts->{speed} = $bwlimit * 1024;
6765         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6766     } else {
6767         print "drive mirror is starting for drive-$drive\n";
6768     }
6769
6770     # if a job already runs for this device we get an error, catch it for cleanup
6771     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
6772     if (my $err = $@) {
6773         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6774         warn "$@\n" if $@;
6775         die "mirroring error: $err\n";
6776     }
6777
6778     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
6779 }
6780
6781 # $completion can be either
6782 # 'complete': wait until all jobs are ready, block-job-complete them (default)
6783 # 'cancel': wait until all jobs are ready, block-job-cancel them
6784 # 'skip': wait until all jobs are ready, return with block jobs in ready state
6785 sub qemu_drive_mirror_monitor {
6786     my ($vmid, $vmiddst, $jobs, $completion, $qga) = @_;
6787
6788     $completion //= 'complete';
6789
6790     eval {
6791         my $err_complete = 0;
6792
6793         while (1) {
6794             die "storage migration timed out\n" if $err_complete > 300;
6795
6796             my $stats = mon_cmd($vmid, "query-block-jobs");
6797
6798             my $running_mirror_jobs = {};
6799             foreach my $stat (@$stats) {
6800                 next if $stat->{type} ne 'mirror';
6801                 $running_mirror_jobs->{$stat->{device}} = $stat;
6802             }
6803
6804             my $readycounter = 0;
6805
6806             foreach my $job (keys %$jobs) {
6807
6808                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6809                     print "$job : finished\n";
6810                     delete $jobs->{$job};
6811                     next;
6812                 }
6813
6814                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6815
6816                 my $busy = $running_mirror_jobs->{$job}->{busy};
6817                 my $ready = $running_mirror_jobs->{$job}->{ready};
6818                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6819                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6820                     my $remaining = $total - $transferred;
6821                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6822
6823                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6824                 }
6825
6826                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6827             }
6828
6829             last if scalar(keys %$jobs) == 0;
6830
6831             if ($readycounter == scalar(keys %$jobs)) {
6832                 print "all mirroring jobs are ready \n";
6833                 last if $completion eq 'skip'; #do the complete later
6834
6835                 if ($vmiddst && $vmiddst != $vmid) {
6836                     my $agent_running = $qga && qga_check_running($vmid);
6837                     if ($agent_running) {
6838                         print "freeze filesystem\n";
6839                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6840                     } else {
6841                         print "suspend vm\n";
6842                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6843                     }
6844
6845                     # if we clone a disk for a new target vm, we don't switch the disk
6846                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6847
6848                     if ($agent_running) {
6849                         print "unfreeze filesystem\n";
6850                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6851                     } else {
6852                         print "resume vm\n";
6853                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6854                     }
6855
6856                     last;
6857                 } else {
6858
6859                     foreach my $job (keys %$jobs) {
6860                         # try to switch the disk if source and destination are on the same guest
6861                         print "$job: Completing block job...\n";
6862
6863                         my $op;
6864                         if ($completion eq 'complete') {
6865                             $op = 'block-job-complete';
6866                         } elsif ($completion eq 'cancel') {
6867                             $op = 'block-job-cancel';
6868                         } else {
6869                             die "invalid completion value: $completion\n";
6870                         }
6871                         eval { mon_cmd($vmid, $op, device => $job) };
6872                         if ($@ =~ m/cannot be completed/) {
6873                             print "$job: Block job cannot be completed, try again.\n";
6874                             $err_complete++;
6875                         }else {
6876                             print "$job: Completed successfully.\n";
6877                             $jobs->{$job}->{complete} = 1;
6878                         }
6879                     }
6880                 }
6881             }
6882             sleep 1;
6883         }
6884     };
6885     my $err = $@;
6886
6887     if ($err) {
6888         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6889         die "mirroring error: $err";
6890     }
6891
6892 }
6893
6894 sub qemu_blockjobs_cancel {
6895     my ($vmid, $jobs) = @_;
6896
6897     foreach my $job (keys %$jobs) {
6898         print "$job: Cancelling block job\n";
6899         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
6900         $jobs->{$job}->{cancel} = 1;
6901     }
6902
6903     while (1) {
6904         my $stats = mon_cmd($vmid, "query-block-jobs");
6905
6906         my $running_jobs = {};
6907         foreach my $stat (@$stats) {
6908             $running_jobs->{$stat->{device}} = $stat;
6909         }
6910
6911         foreach my $job (keys %$jobs) {
6912
6913             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6914                 print "$job: Done.\n";
6915                 delete $jobs->{$job};
6916             }
6917         }
6918
6919         last if scalar(keys %$jobs) == 0;
6920
6921         sleep 1;
6922     }
6923 }
6924
6925 sub clone_disk {
6926     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6927         $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
6928
6929     my $newvolid;
6930
6931     if (!$full) {
6932         print "create linked clone of drive $drivename ($drive->{file})\n";
6933         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6934         push @$newvollist, $newvolid;
6935     } else {
6936
6937         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6938         $storeid = $storage if $storage;
6939
6940         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6941
6942         print "create full clone of drive $drivename ($drive->{file})\n";
6943         my $name = undef;
6944         my $size = undef;
6945         if (drive_is_cloudinit($drive)) {
6946             $name = "vm-$newvmid-cloudinit";
6947             $name .= ".$dst_format" if $dst_format ne 'raw';
6948             $snapname = undef;
6949             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
6950         } elsif ($drivename eq 'efidisk0') {
6951             $size = get_efivars_size($conf);
6952         } else {
6953             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6954         }
6955         $size /= 1024;
6956         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, $size);
6957         push @$newvollist, $newvolid;
6958
6959         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6960
6961         if (drive_is_cloudinit($drive)) {
6962             goto no_data_clone;
6963         }
6964
6965         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6966         if (!$running || $snapname) {
6967             # TODO: handle bwlimits
6968             if ($drivename eq 'efidisk0') {
6969                 # the relevant data on the efidisk may be smaller than the source
6970                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
6971                 # that is given by the OVMF_VARS.fd
6972                 my $src_path = PVE::Storage::path($storecfg, $drive->{file});
6973                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
6974                 run_command(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=1", "count=$size",
6975                     "if=$src_path", "of=$dst_path"]);
6976             } else {
6977                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6978             }
6979         } else {
6980
6981             my $kvmver = get_running_qemu_version ($vmid);
6982             if (!min_version($kvmver, 2, 7)) {
6983                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6984                     if $drive->{iothread};
6985             }
6986
6987             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs,
6988                 $completion, $qga, $bwlimit);
6989         }
6990     }
6991
6992 no_data_clone:
6993     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6994
6995     my $disk = $drive;
6996     $disk->{format} = undef;
6997     $disk->{file} = $newvolid;
6998     $disk->{size} = $size;
6999
7000     return $disk;
7001 }
7002
7003 sub get_running_qemu_version {
7004     my ($vmid) = @_;
7005     my $res = mon_cmd($vmid, "query-version");
7006     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7007 }
7008
7009 sub qemu_use_old_bios_files {
7010     my ($machine_type) = @_;
7011
7012     return if !$machine_type;
7013
7014     my $use_old_bios_files = undef;
7015
7016     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7017         $machine_type = $1;
7018         $use_old_bios_files = 1;
7019     } else {
7020         my $version = extract_version($machine_type, kvm_user_version());
7021         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7022         # load new efi bios files on migration. So this hack is required to allow
7023         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7024         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7025         $use_old_bios_files = !min_version($version, 2, 4);
7026     }
7027
7028     return ($use_old_bios_files, $machine_type);
7029 }
7030
7031 sub get_efivars_size {
7032     my ($conf) = @_;
7033     my $arch = get_vm_arch($conf);
7034     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7035     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
7036     return -s $ovmf_vars;
7037 }
7038
7039 sub update_efidisk_size {
7040     my ($conf) = @_;
7041
7042     return if !defined($conf->{efidisk0});
7043
7044     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
7045     $disk->{size} = get_efivars_size($conf);
7046     $conf->{efidisk0} = print_drive($disk);
7047
7048     return;
7049 }
7050
7051 sub create_efidisk($$$$$) {
7052     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7053
7054     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7055     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7056
7057     my $vars_size_b = -s $ovmf_vars;
7058     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7059     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7060     PVE::Storage::activate_volumes($storecfg, [$volid]);
7061
7062     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7063     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7064
7065     return ($volid, $size/1024);
7066 }
7067
7068 sub vm_iothreads_list {
7069     my ($vmid) = @_;
7070
7071     my $res = mon_cmd($vmid, 'query-iothreads');
7072
7073     my $iothreads = {};
7074     foreach my $iothread (@$res) {
7075         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7076     }
7077
7078     return $iothreads;
7079 }
7080
7081 sub scsihw_infos {
7082     my ($conf, $drive) = @_;
7083
7084     my $maxdev = 0;
7085
7086     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7087         $maxdev = 7;
7088     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7089         $maxdev = 1;
7090     } else {
7091         $maxdev = 256;
7092     }
7093
7094     my $controller = int($drive->{index} / $maxdev);
7095     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
7096         ? "virtioscsi"
7097         : "scsihw";
7098
7099     return ($maxdev, $controller, $controller_prefix);
7100 }
7101
7102 sub windows_version {
7103     my ($ostype) = @_;
7104
7105     return 0 if !$ostype;
7106
7107     my $winversion = 0;
7108
7109     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7110         $winversion = 5;
7111     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7112         $winversion = 6;
7113     } elsif ($ostype =~ m/^win(\d+)$/) {
7114         $winversion = $1;
7115     }
7116
7117     return $winversion;
7118 }
7119
7120 sub resolve_dst_disk_format {
7121         my ($storecfg, $storeid, $src_volname, $format) = @_;
7122         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7123
7124         if (!$format) {
7125             # if no target format is specified, use the source disk format as hint
7126             if ($src_volname) {
7127                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7128                 $format = qemu_img_format($scfg, $src_volname);
7129             } else {
7130                 return $defFormat;
7131             }
7132         }
7133
7134         # test if requested format is supported - else use default
7135         my $supported = grep { $_ eq $format } @$validFormats;
7136         $format = $defFormat if !$supported;
7137         return $format;
7138 }
7139
7140 # NOTE: if this logic changes, please update docs & possibly gui logic
7141 sub find_vmstate_storage {
7142     my ($conf, $storecfg) = @_;
7143
7144     # first, return storage from conf if set
7145     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7146
7147     my ($target, $shared, $local);
7148
7149     foreach_storage_used_by_vm($conf, sub {
7150         my ($sid) = @_;
7151         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7152         my $dst = $scfg->{shared} ? \$shared : \$local;
7153         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7154     });
7155
7156     # second, use shared storage where VM has at least one disk
7157     # third, use local storage where VM has at least one disk
7158     # fall back to local storage
7159     $target = $shared // $local // 'local';
7160
7161     return $target;
7162 }
7163
7164 sub generate_uuid {
7165     my ($uuid, $uuid_str);
7166     UUID::generate($uuid);
7167     UUID::unparse($uuid, $uuid_str);
7168     return $uuid_str;
7169 }
7170
7171 sub generate_smbios1_uuid {
7172     return "uuid=".generate_uuid();
7173 }
7174
7175 sub nbd_stop {
7176     my ($vmid) = @_;
7177
7178     mon_cmd($vmid, 'nbd-server-stop');
7179 }
7180
7181 sub create_reboot_request {
7182     my ($vmid) = @_;
7183     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7184         or die "failed to create reboot trigger file: $!\n";
7185     close($fh);
7186 }
7187
7188 sub clear_reboot_request {
7189     my ($vmid) = @_;
7190     my $path = "/run/qemu-server/$vmid.reboot";
7191     my $res = 0;
7192
7193     $res = unlink($path);
7194     die "could not remove reboot request for $vmid: $!"
7195         if !$res && $! != POSIX::ENOENT;
7196
7197     return $res;
7198 }
7199
7200 sub bootorder_from_legacy {
7201     my ($conf, $bootcfg) = @_;
7202
7203     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
7204     my $bootindex_hash = {};
7205     my $i = 1;
7206     foreach my $o (split(//, $boot)) {
7207         $bootindex_hash->{$o} = $i*100;
7208         $i++;
7209     }
7210
7211     my $bootorder = {};
7212
7213     PVE::QemuConfig->foreach_volume($conf, sub {
7214         my ($ds, $drive) = @_;
7215
7216         if (drive_is_cdrom ($drive, 1)) {
7217             if ($bootindex_hash->{d}) {
7218                 $bootorder->{$ds} = $bootindex_hash->{d};
7219                 $bootindex_hash->{d} += 1;
7220             }
7221         } elsif ($bootindex_hash->{c}) {
7222             $bootorder->{$ds} = $bootindex_hash->{c}
7223                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
7224             $bootindex_hash->{c} += 1;
7225         }
7226     });
7227
7228     if ($bootindex_hash->{n}) {
7229         for (my $i = 0; $i < $MAX_NETS; $i++) {
7230             my $netname = "net$i";
7231             next if !$conf->{$netname};
7232             $bootorder->{$netname} = $bootindex_hash->{n};
7233             $bootindex_hash->{n} += 1;
7234         }
7235     }
7236
7237     return $bootorder;
7238 }
7239
7240 # Generate default device list for 'boot: order=' property. Matches legacy
7241 # default boot order, but with explicit device names. This is important, since
7242 # the fallback for when neither 'order' nor the old format is specified relies
7243 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
7244 sub get_default_bootdevices {
7245     my ($conf) = @_;
7246
7247     my @ret = ();
7248
7249     # harddisk
7250     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
7251     push @ret, $first if $first;
7252
7253     # cdrom
7254     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
7255     push @ret, $first if $first;
7256
7257     # network
7258     for (my $i = 0; $i < $MAX_NETS; $i++) {
7259         my $netname = "net$i";
7260         next if !$conf->{$netname};
7261         push @ret, $netname;
7262         last;
7263     }
7264
7265     return \@ret;
7266 }
7267
7268 sub device_bootorder {
7269     my ($conf) = @_;
7270
7271     return bootorder_from_legacy($conf) if !defined($conf->{boot});
7272
7273     my $boot = parse_property_string($boot_fmt, $conf->{boot});
7274
7275     my $bootorder = {};
7276     if (!defined($boot) || $boot->{legacy}) {
7277         $bootorder = bootorder_from_legacy($conf, $boot);
7278     } elsif ($boot->{order}) {
7279         my $i = 100; # start at 100 to allow user to insert devices before us with -args
7280         for my $dev (PVE::Tools::split_list($boot->{order})) {
7281             $bootorder->{$dev} = $i++;
7282         }
7283     }
7284
7285     return $bootorder;
7286 }
7287
7288 # bash completion helper
7289
7290 sub complete_backup_archives {
7291     my ($cmdname, $pname, $cvalue) = @_;
7292
7293     my $cfg = PVE::Storage::config();
7294
7295     my $storeid;
7296
7297     if ($cvalue =~ m/^([^:]+):/) {
7298         $storeid = $1;
7299     }
7300
7301     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7302
7303     my $res = [];
7304     foreach my $id (keys %$data) {
7305         foreach my $item (@{$data->{$id}}) {
7306             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
7307             push @$res, $item->{volid} if defined($item->{volid});
7308         }
7309     }
7310
7311     return $res;
7312 }
7313
7314 my $complete_vmid_full = sub {
7315     my ($running) = @_;
7316
7317     my $idlist = vmstatus();
7318
7319     my $res = [];
7320
7321     foreach my $id (keys %$idlist) {
7322         my $d = $idlist->{$id};
7323         if (defined($running)) {
7324             next if $d->{template};
7325             next if $running && $d->{status} ne 'running';
7326             next if !$running && $d->{status} eq 'running';
7327         }
7328         push @$res, $id;
7329
7330     }
7331     return $res;
7332 };
7333
7334 sub complete_vmid {
7335     return &$complete_vmid_full();
7336 }
7337
7338 sub complete_vmid_stopped {
7339     return &$complete_vmid_full(0);
7340 }
7341
7342 sub complete_vmid_running {
7343     return &$complete_vmid_full(1);
7344 }
7345
7346 sub complete_storage {
7347
7348     my $cfg = PVE::Storage::config();
7349     my $ids = $cfg->{ids};
7350
7351     my $res = [];
7352     foreach my $sid (keys %$ids) {
7353         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7354         next if !$ids->{$sid}->{content}->{images};
7355         push @$res, $sid;
7356     }
7357
7358     return $res;
7359 }
7360
7361 sub complete_migration_storage {
7362     my ($cmd, $param, $current_value, $all_args) = @_;
7363
7364     my $targetnode = @$all_args[1];
7365
7366     my $cfg = PVE::Storage::config();
7367     my $ids = $cfg->{ids};
7368
7369     my $res = [];
7370     foreach my $sid (keys %$ids) {
7371         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7372         next if !$ids->{$sid}->{content}->{images};
7373         push @$res, $sid;
7374     }
7375
7376     return $res;
7377 }
7378
7379 1;