PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::DataCenterConfig;
  31 use PVE::Exception qw(raise raise_param_exc);
  32 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  33 use PVE::INotify;
  34 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  35 use PVE::ProcFSTools;
  36 use PVE::RPCEnvironment;
  37 use PVE::Storage;
  38 use PVE::SysFSTools;
  39 use PVE::Systemd;
  40 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  41
  42 use PVE::QMPClient;
  43 use PVE::QemuConfig;
  44 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  45 use PVE::QemuServer::Cloudinit;
  46 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  47 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom parse_drive print_drive);
  48 use PVE::QemuServer::Machine;
  49 use PVE::QemuServer::Memory;
  50 use PVE::QemuServer::Monitor qw(mon_cmd);
  51 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  52 use PVE::QemuServer::USB qw(parse_usb_device);
  53
  54 my $have_sdn;
  55 eval {
  56     require PVE::Network::SDN::Zones;
  57     $have_sdn = 1;
  58 };
  59
  60 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  61 my $OVMF = {
  62     x86_64 => [
  63         "$EDK2_FW_BASE/OVMF_CODE.fd",
  64         "$EDK2_FW_BASE/OVMF_VARS.fd"
  65     ],
  66     aarch64 => [
  67         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  68         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  69     ],
  70 };
  71
  72 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  73
  74 # Note about locking: we use flock on the config file protect
  75 # against concurent actions.
  76 # Aditionaly, we have a 'lock' setting in the config file. This
  77 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  78 # allowed when such lock is set. But you can ignore this kind of
  79 # lock with the --skiplock flag.
  80
  81 cfs_register_file('/qemu-server/',
  82                   \&parse_vm_config,
  83                   \&write_vm_config);
  84
  85 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  86     description => "Some command save/restore state from this location.",
  87     type => 'string',
  88     maxLength => 128,
  89     optional => 1,
  90 });
  91
  92 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  93         description => "Specifies the Qemu machine type.",
  94         type => 'string',
  95         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
  96         maxLength => 40,
  97         optional => 1,
  98 });
  99
 100
 101 sub map_storage {
 102     my ($map, $source) = @_;
 103
 104     return $source if !defined($map);
 105
 106     return $map->{entries}->{$source}
 107         if $map->{entries} && defined($map->{entries}->{$source});
 108
 109     return $map->{default} if $map->{default};
 110
 111     # identity (fallback)
 112     return $source;
 113 }
 114
 115 PVE::JSONSchema::register_standard_option('pve-targetstorage', {
 116     description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
 117     type => 'string',
 118     format => 'storagepair-list',
 119     optional => 1,
 120 });
 121
 122 #no warnings 'redefine';
 123
 124 sub cgroups_write {
 125    my ($controller, $vmid, $option, $value) = @_;
 126
 127    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 128    PVE::ProcFSTools::write_proc_entry($path, $value);
 129
 130 }
 131
 132 my $nodename_cache;
 133 sub nodename {
 134     $nodename_cache //= PVE::INotify::nodename();
 135     return $nodename_cache;
 136 }
 137
 138 my $watchdog_fmt = {
 139     model => {
 140         default_key => 1,
 141         type => 'string',
 142         enum => [qw(i6300esb ib700)],
 143         description => "Watchdog type to emulate.",
 144         default => 'i6300esb',
 145         optional => 1,
 146     },
 147     action => {
 148         type => 'string',
 149         enum => [qw(reset shutdown poweroff pause debug none)],
 150         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 151         optional => 1,
 152     },
 153 };
 154 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 155
 156 my $agent_fmt = {
 157     enabled => {
 158         description => "Enable/disable Qemu GuestAgent.",
 159         type => 'boolean',
 160         default => 0,
 161         default_key => 1,
 162     },
 163     fstrim_cloned_disks => {
 164         description => "Run fstrim after cloning/moving a disk.",
 165         type => 'boolean',
 166         optional => 1,
 167         default => 0
 168     },
 169     type => {
 170         description => "Select the agent type",
 171         type => 'string',
 172         default => 'virtio',
 173         optional => 1,
 174         enum => [qw(virtio isa)],
 175     },
 176 };
 177
 178 my $vga_fmt = {
 179     type => {
 180         description => "Select the VGA type.",
 181         type => 'string',
 182         default => 'std',
 183         optional => 1,
 184         default_key => 1,
 185         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 186     },
 187     memory => {
 188         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 189         type => 'integer',
 190         optional => 1,
 191         minimum => 4,
 192         maximum => 512,
 193     },
 194 };
 195
 196 my $ivshmem_fmt = {
 197     size => {
 198         type => 'integer',
 199         minimum => 1,
 200         description => "The size of the file in MB.",
 201     },
 202     name => {
 203         type => 'string',
 204         pattern => '[a-zA-Z0-9\-]+',
 205         optional => 1,
 206         format_description => 'string',
 207         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 208     },
 209 };
 210
 211 my $audio_fmt = {
 212     device => {
 213         type => 'string',
 214         enum => [qw(ich9-intel-hda intel-hda AC97)],
 215         description =>  "Configure an audio device."
 216     },
 217     driver =>  {
 218         type => 'string',
 219         enum => ['spice'],
 220         default => 'spice',
 221         optional => 1,
 222         description => "Driver backend for the audio device."
 223     },
 224 };
 225
 226 my $spice_enhancements_fmt = {
 227     foldersharing => {
 228         type => 'boolean',
 229         optional => 1,
 230         default => '0',
 231         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 232     },
 233     videostreaming =>  {
 234         type => 'string',
 235         enum => ['off', 'all', 'filter'],
 236         default => 'off',
 237         optional => 1,
 238         description => "Enable video streaming. Uses compression for detected video streams."
 239     },
 240 };
 241
 242 my $rng_fmt = {
 243     source => {
 244         type => 'string',
 245         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 246         default_key => 1,
 247         description => "The file on the host to gather entropy from. In most"
 248                      . " cases /dev/urandom should be preferred over /dev/random"
 249                      . " to avoid entropy-starvation issues on the host. Using"
 250                      . " urandom does *not* decrease security in any meaningful"
 251                      . " way, as it's still seeded from real entropy, and the"
 252                      . " bytes provided will most likely be mixed with real"
 253                      . " entropy on the guest as well. /dev/hwrng can be used"
 254                      . " to pass through a hardware RNG from the host.",
 255     },
 256     max_bytes => {
 257         type => 'integer',
 258         description => "Maximum bytes of entropy injected into the guest every"
 259                      . " 'period' milliseconds. Prefer a lower value when using"
 260                      . " /dev/random as source. Use 0 to disable limiting"
 261                      . " (potentially dangerous!).",
 262         optional => 1,
 263
 264         # default is 1 KiB/s, provides enough entropy to the guest to avoid
 265         # boot-starvation issues (e.g. systemd etc...) while allowing no chance
 266         # of overwhelming the host, provided we're reading from /dev/urandom
 267         default => 1024,
 268     },
 269     period => {
 270         type => 'integer',
 271         description => "Every 'period' milliseconds the entropy-injection quota"
 272                      . " is reset, allowing the guest to retrieve another"
 273                      . " 'max_bytes' of entropy.",
 274         optional => 1,
 275         default => 1000,
 276     },
 277 };
 278
 279 my $confdesc = {
 280     onboot => {
 281         optional => 1,
 282         type => 'boolean',
 283         description => "Specifies whether a VM will be started during system bootup.",
 284         default => 0,
 285     },
 286     autostart => {
 287         optional => 1,
 288         type => 'boolean',
 289         description => "Automatic restart after crash (currently ignored).",
 290         default => 0,
 291     },
 292     hotplug => {
 293         optional => 1,
 294         type => 'string', format => 'pve-hotplug-features',
 295         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 296         default => 'network,disk,usb',
 297     },
 298     reboot => {
 299         optional => 1,
 300         type => 'boolean',
 301         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 302         default => 1,
 303     },
 304     lock => {
 305         optional => 1,
 306         type => 'string',
 307         description => "Lock/unlock the VM.",
 308         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 309     },
 310     cpulimit => {
 311         optional => 1,
 312         type => 'number',
 313         description => "Limit of CPU usage.",
 314         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 315         minimum => 0,
 316         maximum => 128,
 317         default => 0,
 318     },
 319     cpuunits => {
 320         optional => 1,
 321         type => 'integer',
 322         description => "CPU weight for a VM.",
 323         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 324         minimum => 2,
 325         maximum => 262144,
 326         default => 1024,
 327     },
 328     memory => {
 329         optional => 1,
 330         type => 'integer',
 331         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 332         minimum => 16,
 333         default => 512,
 334     },
 335     balloon => {
 336         optional => 1,
 337         type => 'integer',
 338         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 339         minimum => 0,
 340     },
 341     shares => {
 342         optional => 1,
 343         type => 'integer',
 344         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 345         minimum => 0,
 346         maximum => 50000,
 347         default => 1000,
 348     },
 349     keyboard => {
 350         optional => 1,
 351         type => 'string',
 352         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 353                        "It should not be necessary to set it.",
 354         enum => PVE::Tools::kvmkeymaplist(),
 355         default => undef,
 356     },
 357     name => {
 358         optional => 1,
 359         type => 'string', format => 'dns-name',
 360         description => "Set a name for the VM. Only used on the configuration web interface.",
 361     },
 362     scsihw => {
 363         optional => 1,
 364         type => 'string',
 365         description => "SCSI controller model",
 366         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 367         default => 'lsi',
 368     },
 369     description => {
 370         optional => 1,
 371         type => 'string',
 372         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 373     },
 374     ostype => {
 375         optional => 1,
 376         type => 'string',
 377         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 378         description => "Specify guest operating system.",
 379         verbose_description => <<EODESC,
 380 Specify guest operating system. This is used to enable special
 381 optimization/features for specific operating systems:
 382
 383 [horizontal]
 384 other;; unspecified OS
 385 wxp;; Microsoft Windows XP
 386 w2k;; Microsoft Windows 2000
 387 w2k3;; Microsoft Windows 2003
 388 w2k8;; Microsoft Windows 2008
 389 wvista;; Microsoft Windows Vista
 390 win7;; Microsoft Windows 7
 391 win8;; Microsoft Windows 8/2012/2012r2
 392 win10;; Microsoft Windows 10/2016
 393 l24;; Linux 2.4 Kernel
 394 l26;; Linux 2.6 - 5.X Kernel
 395 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 396 EODESC
 397     },
 398     boot => {
 399         optional => 1,
 400         type => 'string', format => 'pve-qm-boot',
 401         description => "Specify guest boot order. Use with 'order=', usage with"
 402                      . " no key or 'legacy=' is deprecated.",
 403     },
 404     bootdisk => {
 405         optional => 1,
 406         type => 'string', format => 'pve-qm-bootdisk',
 407         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 408         pattern => '(ide|sata|scsi|virtio)\d+',
 409     },
 410     smp => {
 411         optional => 1,
 412         type => 'integer',
 413         description => "The number of CPUs. Please use option -sockets instead.",
 414         minimum => 1,
 415         default => 1,
 416     },
 417     sockets => {
 418         optional => 1,
 419         type => 'integer',
 420         description => "The number of CPU sockets.",
 421         minimum => 1,
 422         default => 1,
 423     },
 424     cores => {
 425         optional => 1,
 426         type => 'integer',
 427         description => "The number of cores per socket.",
 428         minimum => 1,
 429         default => 1,
 430     },
 431     numa => {
 432         optional => 1,
 433         type => 'boolean',
 434         description => "Enable/disable NUMA.",
 435         default => 0,
 436     },
 437     hugepages => {
 438         optional => 1,
 439         type => 'string',
 440         description => "Enable/disable hugepages memory.",
 441         enum => [qw(any 2 1024)],
 442     },
 443     keephugepages => {
 444         optional => 1,
 445         type => 'boolean',
 446         default => 0,
 447         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 448             ." after VM shutdown and can be used for subsequent starts.",
 449     },
 450     vcpus => {
 451         optional => 1,
 452         type => 'integer',
 453         description => "Number of hotplugged vcpus.",
 454         minimum => 1,
 455         default => 0,
 456     },
 457     acpi => {
 458         optional => 1,
 459         type => 'boolean',
 460         description => "Enable/disable ACPI.",
 461         default => 1,
 462     },
 463     agent => {
 464         optional => 1,
 465         description => "Enable/disable Qemu GuestAgent and its properties.",
 466         type => 'string',
 467         format => $agent_fmt,
 468     },
 469     kvm => {
 470         optional => 1,
 471         type => 'boolean',
 472         description => "Enable/disable KVM hardware virtualization.",
 473         default => 1,
 474     },
 475     tdf => {
 476         optional => 1,
 477         type => 'boolean',
 478         description => "Enable/disable time drift fix.",
 479         default => 0,
 480     },
 481     localtime => {
 482         optional => 1,
 483         type => 'boolean',
 484         description => "Set the real time clock to local time. This is enabled by default if ostype"
 485             ." indicates a Microsoft OS.",
 486     },
 487     freeze => {
 488         optional => 1,
 489         type => 'boolean',
 490         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 491     },
 492     vga => {
 493         optional => 1,
 494         type => 'string', format => $vga_fmt,
 495         description => "Configure the VGA hardware.",
 496         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 497             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 498             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 499             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 500             ." display server. For win* OS you can select how many independent displays you want,"
 501             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 502             ." using a serial device as terminal.",
 503     },
 504     watchdog => {
 505         optional => 1,
 506         type => 'string', format => 'pve-qm-watchdog',
 507         description => "Create a virtual hardware watchdog device.",
 508         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 509             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 510             ." else the watchdog will reset the guest (or execute the respective action specified)",
 511     },
 512     startdate => {
 513         optional => 1,
 514         type => 'string',
 515         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 516         description => "Set the initial date of the real time clock. Valid format for date are:"
 517             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 518         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 519         default => 'now',
 520     },
 521     startup =>  get_standard_option('pve-startup-order'),
 522     template => {
 523         optional => 1,
 524         type => 'boolean',
 525         description => "Enable/disable Template.",
 526         default => 0,
 527     },
 528     args => {
 529         optional => 1,
 530         type => 'string',
 531         description => "Arbitrary arguments passed to kvm.",
 532         verbose_description => <<EODESCR,
 533 Arbitrary arguments passed to kvm, for example:
 534
 535 args: -no-reboot -no-hpet
 536
 537 NOTE: this option is for experts only.
 538 EODESCR
 539     },
 540     tablet => {
 541         optional => 1,
 542         type => 'boolean',
 543         default => 1,
 544         description => "Enable/disable the USB tablet device.",
 545         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 546             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 547             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 548             ." may consider disabling this to save some context switches. This is turned off by"
 549             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 550     },
 551     migrate_speed => {
 552         optional => 1,
 553         type => 'integer',
 554         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 555         minimum => 0,
 556         default => 0,
 557     },
 558     migrate_downtime => {
 559         optional => 1,
 560         type => 'number',
 561         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 562         minimum => 0,
 563         default => 0.1,
 564     },
 565     cdrom => {
 566         optional => 1,
 567         type => 'string', format => 'pve-qm-ide',
 568         typetext => '<volume>',
 569         description => "This is an alias for option -ide2",
 570     },
 571     cpu => {
 572         optional => 1,
 573         description => "Emulated CPU type.",
 574         type => 'string',
 575         format => 'pve-vm-cpu-conf',
 576     },
 577     parent => get_standard_option('pve-snapshot-name', {
 578         optional => 1,
 579         description => "Parent snapshot name. This is used internally, and should not be modified.",
 580     }),
 581     snaptime => {
 582         optional => 1,
 583         description => "Timestamp for snapshots.",
 584         type => 'integer',
 585         minimum => 0,
 586     },
 587     vmstate => {
 588         optional => 1,
 589         type => 'string', format => 'pve-volume-id',
 590         description => "Reference to a volume which stores the VM state. This is used internally"
 591             ." for snapshots.",
 592     },
 593     vmstatestorage => get_standard_option('pve-storage-id', {
 594         description => "Default storage for VM state volumes/files.",
 595         optional => 1,
 596     }),
 597     runningmachine => get_standard_option('pve-qemu-machine', {
 598         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 599             ." for snapshots.",
 600     }),
 601     runningcpu => {
 602         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 603             ." internally for snapshots.",
 604         optional => 1,
 605         type => 'string',
 606         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 607         format_description => 'QEMU -cpu parameter'
 608     },
 609     machine => get_standard_option('pve-qemu-machine'),
 610     arch => {
 611         description => "Virtual processor architecture. Defaults to the host.",
 612         optional => 1,
 613         type => 'string',
 614         enum => [qw(x86_64 aarch64)],
 615     },
 616     smbios1 => {
 617         description => "Specify SMBIOS type 1 fields.",
 618         type => 'string', format => 'pve-qm-smbios1',
 619         maxLength => 512,
 620         optional => 1,
 621     },
 622     protection => {
 623         optional => 1,
 624         type => 'boolean',
 625         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 626             ." remove disk operations.",
 627         default => 0,
 628     },
 629     bios => {
 630         optional => 1,
 631         type => 'string',
 632         enum => [ qw(seabios ovmf) ],
 633         description => "Select BIOS implementation.",
 634         default => 'seabios',
 635     },
 636     vmgenid => {
 637         type => 'string',
 638         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 639         format_description => 'UUID',
 640         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 641             ." to disable explicitly.",
 642         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 643             ." value identifier to the guest OS. This allows to notify the guest operating system"
 644             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 645             ." execution or creation from a template). The guest operating system notices the"
 646             ." change, and is then able to react as appropriate by marking its copies of"
 647             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 648             ."Note that auto-creation only works when done through API/CLI create or update methods"
 649             .", but not when manually editing the config file.",
 650         default => "1 (autogenerated)",
 651         optional => 1,
 652     },
 653     hookscript => {
 654         type => 'string',
 655         format => 'pve-volume-id',
 656         optional => 1,
 657         description => "Script that will be executed during various steps in the vms lifetime.",
 658     },
 659     ivshmem => {
 660         type => 'string',
 661         format => $ivshmem_fmt,
 662         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 663             ." the host.",
 664         optional => 1,
 665     },
 666     audio0 => {
 667         type => 'string',
 668         format => $audio_fmt,
 669         description => "Configure a audio device, useful in combination with QXL/Spice.",
 670         optional => 1
 671     },
 672     spice_enhancements => {
 673         type => 'string',
 674         format => $spice_enhancements_fmt,
 675         description => "Configure additional enhancements for SPICE.",
 676         optional => 1
 677     },
 678     tags => {
 679         type => 'string', format => 'pve-tag-list',
 680         description => 'Tags of the VM. This is only meta information.',
 681         optional => 1,
 682     },
 683     rng0 => {
 684         type => 'string',
 685         format => $rng_fmt,
 686         description => "Configure a VirtIO-based Random Number Generator.",
 687         optional => 1,
 688     },
 689 };
 690
 691 my $cicustom_fmt = {
 692     meta => {
 693         type => 'string',
 694         optional => 1,
 695         description => 'Specify a custom file containing all meta data passed to the VM via"
 696             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 697         format => 'pve-volume-id',
 698         format_description => 'volume',
 699     },
 700     network => {
 701         type => 'string',
 702         optional => 1,
 703         description => 'Specify a custom file containing all network data passed to the VM via'
 704             .' cloud-init.',
 705         format => 'pve-volume-id',
 706         format_description => 'volume',
 707     },
 708     user => {
 709         type => 'string',
 710         optional => 1,
 711         description => 'Specify a custom file containing all user data passed to the VM via'
 712             .' cloud-init.',
 713         format => 'pve-volume-id',
 714         format_description => 'volume',
 715     },
 716 };
 717 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 718
 719 my $confdesc_cloudinit = {
 720     citype => {
 721         optional => 1,
 722         type => 'string',
 723         description => 'Specifies the cloud-init configuration format. The default depends on the'
 724             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 725             .' and `configdrive2` for windows.',
 726         enum => ['configdrive2', 'nocloud'],
 727     },
 728     ciuser => {
 729         optional => 1,
 730         type => 'string',
 731         description => "cloud-init: User name to change ssh keys and password for instead of the"
 732             ." image's configured default user.",
 733     },
 734     cipassword => {
 735         optional => 1,
 736         type => 'string',
 737         description => 'cloud-init: Password to assign the user. Using this is generally not'
 738             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 739             .' support hashed passwords.',
 740     },
 741     cicustom => {
 742         optional => 1,
 743         type => 'string',
 744         description => 'cloud-init: Specify custom files to replace the automatically generated'
 745             .' ones at start.',
 746         format => 'pve-qm-cicustom',
 747     },
 748     searchdomain => {
 749         optional => 1,
 750         type => 'string',
 751         description => "cloud-init: Sets DNS search domains for a container. Create will'
 752             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 753             .' are set.",
 754     },
 755     nameserver => {
 756         optional => 1,
 757         type => 'string', format => 'address-list',
 758         description => "cloud-init: Sets DNS server IP address for a container. Create will'
 759             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 760             .' are set.",
 761     },
 762     sshkeys => {
 763         optional => 1,
 764         type => 'string',
 765         format => 'urlencoded',
 766         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 767     },
 768 };
 769
 770 # what about other qemu settings ?
 771 #cpu => 'string',
 772 #machine => 'string',
 773 #fda => 'file',
 774 #fdb => 'file',
 775 #mtdblock => 'file',
 776 #sd => 'file',
 777 #pflash => 'file',
 778 #snapshot => 'bool',
 779 #bootp => 'file',
 780 ##tftp => 'dir',
 781 ##smb => 'dir',
 782 #kernel => 'file',
 783 #append => 'string',
 784 #initrd => 'file',
 785 ##soundhw => 'string',
 786
 787 while (my ($k, $v) = each %$confdesc) {
 788     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 789 }
 790
 791 my $MAX_USB_DEVICES = 5;
 792 my $MAX_NETS = 32;
 793 my $MAX_SERIAL_PORTS = 4;
 794 my $MAX_PARALLEL_PORTS = 3;
 795 my $MAX_NUMA = 8;
 796
 797 my $numa_fmt = {
 798     cpus => {
 799         type => "string",
 800         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 801         description => "CPUs accessing this NUMA node.",
 802         format_description => "id[-id];...",
 803     },
 804     memory => {
 805         type => "number",
 806         description => "Amount of memory this NUMA node provides.",
 807         optional => 1,
 808     },
 809     hostnodes => {
 810         type => "string",
 811         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 812         description => "Host NUMA nodes to use.",
 813         format_description => "id[-id];...",
 814         optional => 1,
 815     },
 816     policy => {
 817         type => 'string',
 818         enum => [qw(preferred bind interleave)],
 819         description => "NUMA allocation policy.",
 820         optional => 1,
 821     },
 822 };
 823 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 824 my $numadesc = {
 825     optional => 1,
 826     type => 'string', format => $numa_fmt,
 827     description => "NUMA topology.",
 828 };
 829 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 830
 831 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 832     $confdesc->{"numa$i"} = $numadesc;
 833 }
 834
 835 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 836                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 837                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 838 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 839
 840 my $net_fmt_bridge_descr = <<__EOD__;
 841 Bridge to attach the network device to. The Proxmox VE standard bridge
 842 is called 'vmbr0'.
 843
 844 If you do not specify a bridge, we create a kvm user (NATed) network
 845 device, which provides DHCP and DNS services. The following addresses
 846 are used:
 847
 848  10.0.2.2   Gateway
 849  10.0.2.3   DNS Server
 850  10.0.2.4   SMB Server
 851
 852 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 853 __EOD__
 854
 855 my $net_fmt = {
 856     macaddr  => get_standard_option('mac-addr', {
 857         description => "MAC address. That address must be unique withing your network. This is"
 858             ." automatically generated if not specified.",
 859     }),
 860     model => {
 861         type => 'string',
 862         description => "Network Card Model. The 'virtio' model provides the best performance with"
 863             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 864             ." best to use 'e1000'.",
 865         enum => $nic_model_list,
 866         default_key => 1,
 867     },
 868     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 869     bridge => {
 870         type => 'string',
 871         description => $net_fmt_bridge_descr,
 872         format_description => 'bridge',
 873         pattern => '[-_.\w\d]+',
 874         optional => 1,
 875     },
 876     queues => {
 877         type => 'integer',
 878         minimum => 0, maximum => 16,
 879         description => 'Number of packet queues to be used on the device.',
 880         optional => 1,
 881     },
 882     rate => {
 883         type => 'number',
 884         minimum => 0,
 885         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 886         optional => 1,
 887     },
 888     tag => {
 889         type => 'integer',
 890         minimum => 1, maximum => 4094,
 891         description => 'VLAN tag to apply to packets on this interface.',
 892         optional => 1,
 893     },
 894     trunks => {
 895         type => 'string',
 896         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 897         description => 'VLAN trunks to pass through this interface.',
 898         format_description => 'vlanid[;vlanid...]',
 899         optional => 1,
 900     },
 901     firewall => {
 902         type => 'boolean',
 903         description => 'Whether this interface should be protected by the firewall.',
 904         optional => 1,
 905     },
 906     link_down => {
 907         type => 'boolean',
 908         description => 'Whether this interface should be disconnected (like pulling the plug).',
 909         optional => 1,
 910     },
 911     mtu => {
 912         type => 'integer',
 913         minimum => 1, maximum => 65520,
 914         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 915         optional => 1,
 916     },
 917 };
 918
 919 my $netdesc = {
 920     optional => 1,
 921     type => 'string', format => $net_fmt,
 922     description => "Specify network devices.",
 923 };
 924
 925 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 926
 927 my $ipconfig_fmt = {
 928     ip => {
 929         type => 'string',
 930         format => 'pve-ipv4-config',
 931         format_description => 'IPv4Format/CIDR',
 932         description => 'IPv4 address in CIDR format.',
 933         optional => 1,
 934         default => 'dhcp',
 935     },
 936     gw => {
 937         type => 'string',
 938         format => 'ipv4',
 939         format_description => 'GatewayIPv4',
 940         description => 'Default gateway for IPv4 traffic.',
 941         optional => 1,
 942         requires => 'ip',
 943     },
 944     ip6 => {
 945         type => 'string',
 946         format => 'pve-ipv6-config',
 947         format_description => 'IPv6Format/CIDR',
 948         description => 'IPv6 address in CIDR format.',
 949         optional => 1,
 950         default => 'dhcp',
 951     },
 952     gw6 => {
 953         type => 'string',
 954         format => 'ipv6',
 955         format_description => 'GatewayIPv6',
 956         description => 'Default gateway for IPv6 traffic.',
 957         optional => 1,
 958         requires => 'ip6',
 959     },
 960 };
 961 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 962 my $ipconfigdesc = {
 963     optional => 1,
 964     type => 'string', format => 'pve-qm-ipconfig',
 965     description => <<'EODESCR',
 966 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 967
 968 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 969
 970 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
 971 gateway should be provided.
 972 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 973
 974 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
 975 dhcp on IPv4.
 976 EODESCR
 977 };
 978 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 979
 980 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 981     $confdesc->{"net$i"} = $netdesc;
 982     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 983 }
 984
 985 foreach my $key (keys %$confdesc_cloudinit) {
 986     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 987 }
 988
 989 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 990 sub verify_volume_id_or_qm_path {
 991     my ($volid, $noerr) = @_;
 992
 993     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 994         return $volid;
 995     }
 996
 997     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 998     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 999     if ($@) {
1000         return if $noerr;
1001         die $@;
1002     }
1003     return $volid;
1004 }
1005
1006 my $usb_fmt = {
1007     host => {
1008         default_key => 1,
1009         type => 'string', format => 'pve-qm-usb-device',
1010         format_description => 'HOSTUSBDEVICE|spice',
1011         description => <<EODESCR,
1012 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1013
1014  'bus-port(.port)*' (decimal numbers) or
1015  'vendor_id:product_id' (hexadeciaml numbers) or
1016  'spice'
1017
1018 You can use the 'lsusb -t' command to list existing usb devices.
1019
1020 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1021 machines - use with special care.
1022
1023 The value 'spice' can be used to add a usb redirection devices for spice.
1024 EODESCR
1025     },
1026     usb3 => {
1027         optional => 1,
1028         type => 'boolean',
1029         description => "Specifies whether if given host option is a USB3 device or port.",
1030         default => 0,
1031     },
1032 };
1033
1034 my $usbdesc = {
1035     optional => 1,
1036     type => 'string', format => $usb_fmt,
1037     description => "Configure an USB device (n is 0 to 4).",
1038 };
1039 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1040
1041 my $serialdesc = {
1042         optional => 1,
1043         type => 'string',
1044         pattern => '(/dev/.+|socket)',
1045         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1046         verbose_description =>  <<EODESCR,
1047 Create a serial device inside the VM (n is 0 to 3), and pass through a
1048 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1049 host side (use 'qm terminal' to open a terminal connection).
1050
1051 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1052 use with special care.
1053
1054 CAUTION: Experimental! User reported problems with this option.
1055 EODESCR
1056 };
1057
1058 my $paralleldesc= {
1059         optional => 1,
1060         type => 'string',
1061         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1062         description =>  "Map host parallel devices (n is 0 to 2).",
1063         verbose_description =>  <<EODESCR,
1064 Map host parallel devices (n is 0 to 2).
1065
1066 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1067 machines - use with special care.
1068
1069 CAUTION: Experimental! User reported problems with this option.
1070 EODESCR
1071 };
1072
1073 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1074     $confdesc->{"parallel$i"} = $paralleldesc;
1075 }
1076
1077 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1078     $confdesc->{"serial$i"} = $serialdesc;
1079 }
1080
1081 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1082     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1083 }
1084
1085 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1086     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1087 }
1088
1089 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1090     $confdesc->{"usb$i"} = $usbdesc;
1091 }
1092
1093 my $boot_fmt = {
1094     legacy => {
1095         optional => 1,
1096         default_key => 1,
1097         type => 'string',
1098         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1099                      . " Deprecated, use 'order=' instead.",
1100         pattern => '[acdn]{1,4}',
1101         format_description => "[acdn]{1,4}",
1102
1103         # note: this is also the fallback if boot: is not given at all
1104         default => 'cdn',
1105     },
1106     order => {
1107         optional => 1,
1108         type => 'string',
1109         format => 'pve-qm-bootdev-list',
1110         format_description => "device[;device...]",
1111         description => <<EODESC,
1112 The guest will attempt to boot from devices in the order they appear here.
1113
1114 Disks, optical drives and passed-through storage USB devices will be directly
1115 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1116 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1117
1118 Note that only devices in this list will be marked as bootable and thus loaded
1119 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1120 (e.g. software-raid), you need to specify all of them here.
1121
1122 Overrides the deprecated 'legacy=[acdn]*' value when given.
1123 EODESC
1124     },
1125 };
1126 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1127
1128 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1129 sub verify_bootdev {
1130     my ($dev, $noerr) = @_;
1131
1132     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && $dev !~ m/^efidisk/;
1133
1134     my $check = sub {
1135         my ($base) = @_;
1136         return 0 if $dev !~ m/^$base\d+$/;
1137         return 0 if !$confdesc->{$dev};
1138         return 1;
1139     };
1140
1141     return $dev if $check->("net");
1142     return $dev if $check->("usb");
1143     return $dev if $check->("hostpci");
1144
1145     return if $noerr;
1146     die "invalid boot device '$dev'\n";
1147 }
1148
1149 sub print_bootorder {
1150     my ($devs) = @_;
1151     return "" if !@$devs;
1152     my $data = { order => join(';', @$devs) };
1153     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1154 }
1155
1156 my $kvm_api_version = 0;
1157
1158 sub kvm_version {
1159     return $kvm_api_version if $kvm_api_version;
1160
1161     open my $fh, '<', '/dev/kvm' or return;
1162
1163     # 0xae00 => KVM_GET_API_VERSION
1164     $kvm_api_version = ioctl($fh, 0xae00, 0);
1165     close($fh);
1166
1167     return $kvm_api_version;
1168 }
1169
1170 my $kvm_user_version = {};
1171 my $kvm_mtime = {};
1172
1173 sub kvm_user_version {
1174     my ($binary) = @_;
1175
1176     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1177     my $st = stat($binary);
1178
1179     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1180     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1181         $cachedmtime == $st->mtime;
1182
1183     $kvm_user_version->{$binary} = 'unknown';
1184     $kvm_mtime->{$binary} = $st->mtime;
1185
1186     my $code = sub {
1187         my $line = shift;
1188         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1189             $kvm_user_version->{$binary} = $2;
1190         }
1191     };
1192
1193     eval { run_command([$binary, '--version'], outfunc => $code); };
1194     warn $@ if $@;
1195
1196     return $kvm_user_version->{$binary};
1197
1198 }
1199 my sub extract_version {
1200     my ($machine_type, $version) = @_;
1201     $version = kvm_user_version() if !defined($version);
1202     PVE::QemuServer::Machine::extract_version($machine_type, $version)
1203 }
1204
1205 sub kernel_has_vhost_net {
1206     return -c '/dev/vhost-net';
1207 }
1208
1209 sub option_exists {
1210     my $key = shift;
1211     return defined($confdesc->{$key});
1212 }
1213
1214 my $cdrom_path;
1215 sub get_cdrom_path {
1216
1217     return  $cdrom_path if $cdrom_path;
1218
1219     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1220     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1221     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1222 }
1223
1224 sub get_iso_path {
1225     my ($storecfg, $vmid, $cdrom) = @_;
1226
1227     if ($cdrom eq 'cdrom') {
1228         return get_cdrom_path();
1229     } elsif ($cdrom eq 'none') {
1230         return '';
1231     } elsif ($cdrom =~ m|^/|) {
1232         return $cdrom;
1233     } else {
1234         return PVE::Storage::path($storecfg, $cdrom);
1235     }
1236 }
1237
1238 # try to convert old style file names to volume IDs
1239 sub filename_to_volume_id {
1240     my ($vmid, $file, $media) = @_;
1241
1242      if (!($file eq 'none' || $file eq 'cdrom' ||
1243           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1244
1245         return if $file =~ m|/|;
1246
1247         if ($media && $media eq 'cdrom') {
1248             $file = "local:iso/$file";
1249         } else {
1250             $file = "local:$vmid/$file";
1251         }
1252     }
1253
1254     return $file;
1255 }
1256
1257 sub verify_media_type {
1258     my ($opt, $vtype, $media) = @_;
1259
1260     return if !$media;
1261
1262     my $etype;
1263     if ($media eq 'disk') {
1264         $etype = 'images';
1265     } elsif ($media eq 'cdrom') {
1266         $etype = 'iso';
1267     } else {
1268         die "internal error";
1269     }
1270
1271     return if ($vtype eq $etype);
1272
1273     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1274 }
1275
1276 sub cleanup_drive_path {
1277     my ($opt, $storecfg, $drive) = @_;
1278
1279     # try to convert filesystem paths to volume IDs
1280
1281     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1282         ($drive->{file} !~ m|^/dev/.+|) &&
1283         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1284         ($drive->{file} !~ m/^\d+$/)) {
1285         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1286         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1287             if !$vtype;
1288         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1289         verify_media_type($opt, $vtype, $drive->{media});
1290         $drive->{file} = $volid;
1291     }
1292
1293     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1294 }
1295
1296 sub parse_hotplug_features {
1297     my ($data) = @_;
1298
1299     my $res = {};
1300
1301     return $res if $data eq '0';
1302
1303     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1304
1305     foreach my $feature (PVE::Tools::split_list($data)) {
1306         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1307             $res->{$1} = 1;
1308         } else {
1309             die "invalid hotplug feature '$feature'\n";
1310         }
1311     }
1312     return $res;
1313 }
1314
1315 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1316 sub pve_verify_hotplug_features {
1317     my ($value, $noerr) = @_;
1318
1319     return $value if parse_hotplug_features($value);
1320
1321     return if $noerr;
1322
1323     die "unable to parse hotplug option\n";
1324 }
1325
1326 sub scsi_inquiry {
1327     my($fh, $noerr) = @_;
1328
1329     my $SG_IO = 0x2285;
1330     my $SG_GET_VERSION_NUM = 0x2282;
1331
1332     my $versionbuf = "\x00" x 8;
1333     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1334     if (!$ret) {
1335         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1336         return;
1337     }
1338     my $version = unpack("I", $versionbuf);
1339     if ($version < 30000) {
1340         die "scsi generic interface too old\n"  if !$noerr;
1341         return;
1342     }
1343
1344     my $buf = "\x00" x 36;
1345     my $sensebuf = "\x00" x 8;
1346     my $cmd = pack("C x3 C x1", 0x12, 36);
1347
1348     # see /usr/include/scsi/sg.h
1349     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1350
1351     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1352                       length($sensebuf), 0, length($buf), $buf,
1353                       $cmd, $sensebuf, 6000);
1354
1355     $ret = ioctl($fh, $SG_IO, $packet);
1356     if (!$ret) {
1357         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1358         return;
1359     }
1360
1361     my @res = unpack($sg_io_hdr_t, $packet);
1362     if ($res[17] || $res[18]) {
1363         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1364         return;
1365     }
1366
1367     my $res = {};
1368     (my $byte0, my $byte1, $res->{vendor},
1369      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1370
1371     $res->{removable} = $byte1 & 128 ? 1 : 0;
1372     $res->{type} = $byte0 & 31;
1373
1374     return $res;
1375 }
1376
1377 sub path_is_scsi {
1378     my ($path) = @_;
1379
1380     my $fh = IO::File->new("+<$path") || return;
1381     my $res = scsi_inquiry($fh, 1);
1382     close($fh);
1383
1384     return $res;
1385 }
1386
1387 sub print_tabletdevice_full {
1388     my ($conf, $arch) = @_;
1389
1390     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1391
1392     # we use uhci for old VMs because tablet driver was buggy in older qemu
1393     my $usbbus;
1394     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1395         $usbbus = 'ehci';
1396     } else {
1397         $usbbus = 'uhci';
1398     }
1399
1400     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1401 }
1402
1403 sub print_keyboarddevice_full {
1404     my ($conf, $arch, $machine) = @_;
1405
1406     return if $arch ne 'aarch64';
1407
1408     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1409 }
1410
1411 sub print_drivedevice_full {
1412     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1413
1414     my $device = '';
1415     my $maxdev = 0;
1416
1417     my $drive_id = "$drive->{interface}$drive->{index}";
1418     if ($drive->{interface} eq 'virtio') {
1419         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1420         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1421         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1422     } elsif ($drive->{interface} eq 'scsi') {
1423
1424         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1425         my $unit = $drive->{index} % $maxdev;
1426         my $devicetype = 'hd';
1427         my $path = '';
1428         if (drive_is_cdrom($drive)) {
1429             $devicetype = 'cd';
1430         } else {
1431             if ($drive->{file} =~ m|^/|) {
1432                 $path = $drive->{file};
1433                 if (my $info = path_is_scsi($path)) {
1434                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1435                         $devicetype = 'block';
1436                     } elsif ($info->{type} == 1) { # tape
1437                         $devicetype = 'generic';
1438                     }
1439                 }
1440             } else {
1441                  $path = PVE::Storage::path($storecfg, $drive->{file});
1442             }
1443
1444             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1445             my $version = extract_version($machine_type, kvm_user_version());
1446             if ($path =~ m/^iscsi\:\/\// &&
1447                !min_version($version, 4, 1)) {
1448                 $devicetype = 'generic';
1449             }
1450         }
1451
1452         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1453             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1454         } else {
1455             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1456                 .",lun=$drive->{index}";
1457         }
1458         $device .= ",drive=drive-$drive_id,id=$drive_id";
1459
1460         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1461             $device .= ",rotation_rate=1";
1462         }
1463         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1464
1465     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1466         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1467         my $controller = int($drive->{index} / $maxdev);
1468         my $unit = $drive->{index} % $maxdev;
1469         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1470
1471         $device = "ide-$devicetype";
1472         if ($drive->{interface} eq 'ide') {
1473             $device .= ",bus=ide.$controller,unit=$unit";
1474         } else {
1475             $device .= ",bus=ahci$controller.$unit";
1476         }
1477         $device .= ",drive=drive-$drive_id,id=$drive_id";
1478
1479         if ($devicetype eq 'hd') {
1480             if (my $model = $drive->{model}) {
1481                 $model = URI::Escape::uri_unescape($model);
1482                 $device .= ",model=$model";
1483             }
1484             if ($drive->{ssd}) {
1485                 $device .= ",rotation_rate=1";
1486             }
1487         }
1488         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1489     } elsif ($drive->{interface} eq 'usb') {
1490         die "implement me";
1491         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1492     } else {
1493         die "unsupported interface type";
1494     }
1495
1496     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1497
1498     if (my $serial = $drive->{serial}) {
1499         $serial = URI::Escape::uri_unescape($serial);
1500         $device .= ",serial=$serial";
1501     }
1502
1503
1504     return $device;
1505 }
1506
1507 sub get_initiator_name {
1508     my $initiator;
1509
1510     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1511     while (defined(my $line = <$fh>)) {
1512         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1513         $initiator = $1;
1514         last;
1515     }
1516     $fh->close();
1517
1518     return $initiator;
1519 }
1520
1521 sub print_drive_commandline_full {
1522     my ($storecfg, $vmid, $drive) = @_;
1523
1524     my $path;
1525     my $volid = $drive->{file};
1526     my $format;
1527
1528     if (drive_is_cdrom($drive)) {
1529         $path = get_iso_path($storecfg, $vmid, $volid);
1530     } else {
1531         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1532         if ($storeid) {
1533             $path = PVE::Storage::path($storecfg, $volid);
1534             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1535             $format = qemu_img_format($scfg, $volname);
1536         } else {
1537             $path = $volid;
1538             $format = "raw";
1539         }
1540    }
1541
1542     my $opts = '';
1543     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1544     foreach my $o (@qemu_drive_options) {
1545         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1546     }
1547
1548     # snapshot only accepts on|off
1549     if (defined($drive->{snapshot})) {
1550         my $v = $drive->{snapshot} ? 'on' : 'off';
1551         $opts .= ",snapshot=$v";
1552     }
1553
1554     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1555         my ($dir, $qmpname) = @$type;
1556         if (my $v = $drive->{"mbps$dir"}) {
1557             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1558         }
1559         if (my $v = $drive->{"mbps${dir}_max"}) {
1560             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1561         }
1562         if (my $v = $drive->{"bps${dir}_max_length"}) {
1563             $opts .= ",throttling.bps$qmpname-max-length=$v";
1564         }
1565         if (my $v = $drive->{"iops${dir}"}) {
1566             $opts .= ",throttling.iops$qmpname=$v";
1567         }
1568         if (my $v = $drive->{"iops${dir}_max"}) {
1569             $opts .= ",throttling.iops$qmpname-max=$v";
1570         }
1571         if (my $v = $drive->{"iops${dir}_max_length"}) {
1572             $opts .= ",throttling.iops$qmpname-max-length=$v";
1573         }
1574     }
1575
1576     $opts .= ",format=$format" if $format && !$drive->{format};
1577
1578     my $cache_direct = 0;
1579
1580     if (my $cache = $drive->{cache}) {
1581         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1582     } elsif (!drive_is_cdrom($drive)) {
1583         $opts .= ",cache=none";
1584         $cache_direct = 1;
1585     }
1586
1587     # aio native works only with O_DIRECT
1588     if (!$drive->{aio}) {
1589         if($cache_direct) {
1590             $opts .= ",aio=native";
1591         } else {
1592             $opts .= ",aio=threads";
1593         }
1594     }
1595
1596     if (!drive_is_cdrom($drive)) {
1597         my $detectzeroes;
1598         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1599             $detectzeroes = 'off';
1600         } elsif ($drive->{discard}) {
1601             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1602         } else {
1603             # This used to be our default with discard not being specified:
1604             $detectzeroes = 'on';
1605         }
1606         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1607     }
1608
1609     my $pathinfo = $path ? "file=$path," : '';
1610
1611     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1612 }
1613
1614 sub print_netdevice_full {
1615     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1616
1617     my $device = $net->{model};
1618     if ($net->{model} eq 'virtio') {
1619          $device = 'virtio-net-pci';
1620      };
1621
1622     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1623     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1624     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1625         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1626         # and out of each queue plus one config interrupt and control vector queue
1627         my $vectors = $net->{queues} * 2 + 2;
1628         $tmpstr .= ",vectors=$vectors,mq=on";
1629     }
1630     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1631
1632     if (my $mtu = $net->{mtu}) {
1633         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1634             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1635             if ($mtu == 1) {
1636                  $mtu = $bridge_mtu;
1637             } elsif ($mtu < 576) {
1638                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1639             } elsif ($mtu > $bridge_mtu) {
1640                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1641             }
1642             $tmpstr .= ",host_mtu=$mtu";
1643         } else {
1644             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1645         }
1646     }
1647
1648     if ($use_old_bios_files) {
1649         my $romfile;
1650         if ($device eq 'virtio-net-pci') {
1651             $romfile = 'pxe-virtio.rom';
1652         } elsif ($device eq 'e1000') {
1653             $romfile = 'pxe-e1000.rom';
1654         } elsif ($device eq 'ne2k') {
1655             $romfile = 'pxe-ne2k_pci.rom';
1656         } elsif ($device eq 'pcnet') {
1657             $romfile = 'pxe-pcnet.rom';
1658         } elsif ($device eq 'rtl8139') {
1659             $romfile = 'pxe-rtl8139.rom';
1660         }
1661         $tmpstr .= ",romfile=$romfile" if $romfile;
1662     }
1663
1664     return $tmpstr;
1665 }
1666
1667 sub print_netdev_full {
1668     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1669
1670     my $i = '';
1671     if ($netid =~ m/^net(\d+)$/) {
1672         $i = int($1);
1673     }
1674
1675     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1676
1677     my $ifname = "tap${vmid}i$i";
1678
1679     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1680     die "interface name '$ifname' is too long (max 15 character)\n"
1681         if length($ifname) >= 16;
1682
1683     my $vhostparam = '';
1684     if (is_native($arch)) {
1685         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1686     }
1687
1688     my $vmname = $conf->{name} || "vm$vmid";
1689
1690     my $netdev = "";
1691     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1692
1693     if ($net->{bridge}) {
1694         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1695             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1696     } else {
1697         $netdev = "type=user,id=$netid,hostname=$vmname";
1698     }
1699
1700     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1701
1702     return $netdev;
1703 }
1704
1705 my $vga_map = {
1706     'cirrus' => 'cirrus-vga',
1707     'std' => 'VGA',
1708     'vmware' => 'vmware-svga',
1709     'virtio' => 'virtio-vga',
1710 };
1711
1712 sub print_vga_device {
1713     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1714
1715     my $type = $vga_map->{$vga->{type}};
1716     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1717         $type = 'virtio-gpu';
1718     }
1719     my $vgamem_mb = $vga->{memory};
1720
1721     my $max_outputs = '';
1722     if ($qxlnum) {
1723         $type = $id ? 'qxl' : 'qxl-vga';
1724
1725         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1726             # set max outputs so linux can have up to 4 qxl displays with one device
1727             if (min_version($machine_version, 4, 1)) {
1728                 $max_outputs = ",max_outputs=4";
1729             }
1730         }
1731     }
1732
1733     die "no devicetype for $vga->{type}\n" if !$type;
1734
1735     my $memory = "";
1736     if ($vgamem_mb) {
1737         if ($vga->{type} eq 'virtio') {
1738             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1739             $memory = ",max_hostmem=$bytes";
1740         } elsif ($qxlnum) {
1741             # from https://www.spice-space.org/multiple-monitors.html
1742             $memory = ",vgamem_mb=$vga->{memory}";
1743             my $ram = $vgamem_mb * 4;
1744             my $vram = $vgamem_mb * 2;
1745             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1746         } else {
1747             $memory = ",vgamem_mb=$vga->{memory}";
1748         }
1749     } elsif ($qxlnum && $id) {
1750         $memory = ",ram_size=67108864,vram_size=33554432";
1751     }
1752
1753     my $edidoff = "";
1754     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1755         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1756     }
1757
1758     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1759     my $vgaid = "vga" . ($id // '');
1760     my $pciaddr;
1761
1762     if ($q35 && $vgaid eq 'vga') {
1763         # the first display uses pcie.0 bus on q35 machines
1764         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1765     } else {
1766         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1767     }
1768
1769     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1770 }
1771
1772 sub parse_number_sets {
1773     my ($set) = @_;
1774     my $res = [];
1775     foreach my $part (split(/;/, $set)) {
1776         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1777             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1778             push @$res, [ $1, $2 ];
1779         } else {
1780             die "invalid range: $part\n";
1781         }
1782     }
1783     return $res;
1784 }
1785
1786 sub parse_numa {
1787     my ($data) = @_;
1788
1789     my $res = parse_property_string($numa_fmt, $data);
1790     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1791     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1792     return $res;
1793 }
1794
1795 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1796 sub parse_net {
1797     my ($data) = @_;
1798
1799     my $res = eval { parse_property_string($net_fmt, $data) };
1800     if ($@) {
1801         warn $@;
1802         return;
1803     }
1804     if (!defined($res->{macaddr})) {
1805         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1806         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1807     }
1808     return $res;
1809 }
1810
1811 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1812 sub parse_ipconfig {
1813     my ($data) = @_;
1814
1815     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1816     if ($@) {
1817         warn $@;
1818         return;
1819     }
1820
1821     if ($res->{gw} && !$res->{ip}) {
1822         warn 'gateway specified without specifying an IP address';
1823         return;
1824     }
1825     if ($res->{gw6} && !$res->{ip6}) {
1826         warn 'IPv6 gateway specified without specifying an IPv6 address';
1827         return;
1828     }
1829     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1830         warn 'gateway specified together with DHCP';
1831         return;
1832     }
1833     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1834         # gw6 + auto/dhcp
1835         warn "IPv6 gateway specified together with $res->{ip6} address";
1836         return;
1837     }
1838
1839     if (!$res->{ip} && !$res->{ip6}) {
1840         return { ip => 'dhcp', ip6 => 'dhcp' };
1841     }
1842
1843     return $res;
1844 }
1845
1846 sub print_net {
1847     my $net = shift;
1848
1849     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1850 }
1851
1852 sub add_random_macs {
1853     my ($settings) = @_;
1854
1855     foreach my $opt (keys %$settings) {
1856         next if $opt !~ m/^net(\d+)$/;
1857         my $net = parse_net($settings->{$opt});
1858         next if !$net;
1859         $settings->{$opt} = print_net($net);
1860     }
1861 }
1862
1863 sub vm_is_volid_owner {
1864     my ($storecfg, $vmid, $volid) = @_;
1865
1866     if ($volid !~  m|^/|) {
1867         my ($path, $owner);
1868         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1869         if ($owner && ($owner == $vmid)) {
1870             return 1;
1871         }
1872     }
1873
1874     return;
1875 }
1876
1877 sub vmconfig_register_unused_drive {
1878     my ($storecfg, $vmid, $conf, $drive) = @_;
1879
1880     if (drive_is_cloudinit($drive)) {
1881         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1882         warn $@ if $@;
1883     } elsif (!drive_is_cdrom($drive)) {
1884         my $volid = $drive->{file};
1885         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1886             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1887         }
1888     }
1889 }
1890
1891 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1892 my $smbios1_fmt = {
1893     uuid => {
1894         type => 'string',
1895         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1896         format_description => 'UUID',
1897         description => "Set SMBIOS1 UUID.",
1898         optional => 1,
1899     },
1900     version => {
1901         type => 'string',
1902         pattern => '[A-Za-z0-9+\/]+={0,2}',
1903         format_description => 'Base64 encoded string',
1904         description => "Set SMBIOS1 version.",
1905         optional => 1,
1906     },
1907     serial => {
1908         type => 'string',
1909         pattern => '[A-Za-z0-9+\/]+={0,2}',
1910         format_description => 'Base64 encoded string',
1911         description => "Set SMBIOS1 serial number.",
1912         optional => 1,
1913     },
1914     manufacturer => {
1915         type => 'string',
1916         pattern => '[A-Za-z0-9+\/]+={0,2}',
1917         format_description => 'Base64 encoded string',
1918         description => "Set SMBIOS1 manufacturer.",
1919         optional => 1,
1920     },
1921     product => {
1922         type => 'string',
1923         pattern => '[A-Za-z0-9+\/]+={0,2}',
1924         format_description => 'Base64 encoded string',
1925         description => "Set SMBIOS1 product ID.",
1926         optional => 1,
1927     },
1928     sku => {
1929         type => 'string',
1930         pattern => '[A-Za-z0-9+\/]+={0,2}',
1931         format_description => 'Base64 encoded string',
1932         description => "Set SMBIOS1 SKU string.",
1933         optional => 1,
1934     },
1935     family => {
1936         type => 'string',
1937         pattern => '[A-Za-z0-9+\/]+={0,2}',
1938         format_description => 'Base64 encoded string',
1939         description => "Set SMBIOS1 family string.",
1940         optional => 1,
1941     },
1942     base64 => {
1943         type => 'boolean',
1944         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
1945         optional => 1,
1946     },
1947 };
1948
1949 sub parse_smbios1 {
1950     my ($data) = @_;
1951
1952     my $res = eval { parse_property_string($smbios1_fmt, $data) };
1953     warn $@ if $@;
1954     return $res;
1955 }
1956
1957 sub print_smbios1 {
1958     my ($smbios1) = @_;
1959     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1960 }
1961
1962 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1963
1964 sub parse_watchdog {
1965     my ($value) = @_;
1966
1967     return if !$value;
1968
1969     my $res = eval { parse_property_string($watchdog_fmt, $value) };
1970     warn $@ if $@;
1971     return $res;
1972 }
1973
1974 sub parse_guest_agent {
1975     my ($value) = @_;
1976
1977     return {} if !defined($value->{agent});
1978
1979     my $res = eval { parse_property_string($agent_fmt, $value->{agent}) };
1980     warn $@ if $@;
1981
1982     # if the agent is disabled ignore the other potentially set properties
1983     return {} if !$res->{enabled};
1984     return $res;
1985 }
1986
1987 sub parse_vga {
1988     my ($value) = @_;
1989
1990     return {} if !$value;
1991     my $res = eval { parse_property_string($vga_fmt, $value) };
1992     warn $@ if $@;
1993     return $res;
1994 }
1995
1996 sub parse_rng {
1997     my ($value) = @_;
1998
1999     return if !$value;
2000
2001     my $res = eval { parse_property_string($rng_fmt, $value) };
2002     warn $@ if $@;
2003     return $res;
2004 }
2005
2006 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2007 sub verify_usb_device {
2008     my ($value, $noerr) = @_;
2009
2010     return $value if parse_usb_device($value);
2011
2012     return if $noerr;
2013
2014     die "unable to parse usb device\n";
2015 }
2016
2017 # add JSON properties for create and set function
2018 sub json_config_properties {
2019     my $prop = shift;
2020
2021     foreach my $opt (keys %$confdesc) {
2022         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' ||
2023             $opt eq 'runningmachine' || $opt eq 'runningcpu';
2024         $prop->{$opt} = $confdesc->{$opt};
2025     }
2026
2027     return $prop;
2028 }
2029
2030 # return copy of $confdesc_cloudinit to generate documentation
2031 sub cloudinit_config_properties {
2032
2033     return dclone($confdesc_cloudinit);
2034 }
2035
2036 sub check_type {
2037     my ($key, $value) = @_;
2038
2039     die "unknown setting '$key'\n" if !$confdesc->{$key};
2040
2041     my $type = $confdesc->{$key}->{type};
2042
2043     if (!defined($value)) {
2044         die "got undefined value\n";
2045     }
2046
2047     if ($value =~ m/[\n\r]/) {
2048         die "property contains a line feed\n";
2049     }
2050
2051     if ($type eq 'boolean') {
2052         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2053         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2054         die "type check ('boolean') failed - got '$value'\n";
2055     } elsif ($type eq 'integer') {
2056         return int($1) if $value =~ m/^(\d+)$/;
2057         die "type check ('integer') failed - got '$value'\n";
2058     } elsif ($type eq 'number') {
2059         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2060         die "type check ('number') failed - got '$value'\n";
2061     } elsif ($type eq 'string') {
2062         if (my $fmt = $confdesc->{$key}->{format}) {
2063             PVE::JSONSchema::check_format($fmt, $value);
2064             return $value;
2065         }
2066         $value =~ s/^\"(.*)\"$/$1/;
2067         return $value;
2068     } else {
2069         die "internal error"
2070     }
2071 }
2072
2073 sub destroy_vm {
2074     my ($storecfg, $vmid, $skiplock, $replacement_conf) = @_;
2075
2076     my $conf = PVE::QemuConfig->load_config($vmid);
2077
2078     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2079
2080     if ($conf->{template}) {
2081         # check if any base image is still used by a linked clone
2082         PVE::QemuConfig->foreach_volume($conf, sub {
2083                 my ($ds, $drive) = @_;
2084                 return if drive_is_cdrom($drive);
2085
2086                 my $volid = $drive->{file};
2087                 return if !$volid || $volid =~ m|^/|;
2088
2089                 die "base volume '$volid' is still in use by linked cloned\n"
2090                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2091
2092         });
2093     }
2094
2095     # only remove disks owned by this VM
2096     PVE::QemuConfig->foreach_volume($conf, sub {
2097         my ($ds, $drive) = @_;
2098         return if drive_is_cdrom($drive, 1);
2099
2100         my $volid = $drive->{file};
2101         return if !$volid || $volid =~ m|^/|;
2102
2103         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2104         return if !$path || !$owner || ($owner != $vmid);
2105
2106         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2107         warn "Could not remove disk '$volid', check manually: $@" if $@;
2108     });
2109
2110     # also remove unused disk
2111     my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2112     PVE::Storage::foreach_volid($vmdisks, sub {
2113         my ($volid, $sid, $volname, $d) = @_;
2114         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2115         warn $@ if $@;
2116     });
2117
2118     if (defined $replacement_conf) {
2119         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2120     } else {
2121         PVE::QemuConfig->destroy_config($vmid);
2122     }
2123 }
2124
2125 sub parse_vm_config {
2126     my ($filename, $raw) = @_;
2127
2128     return if !defined($raw);
2129
2130     my $res = {
2131         digest => Digest::SHA::sha1_hex($raw),
2132         snapshots => {},
2133         pending => {},
2134     };
2135
2136     $filename =~ m|/qemu-server/(\d+)\.conf$|
2137         || die "got strange filename '$filename'";
2138
2139     my $vmid = $1;
2140
2141     my $conf = $res;
2142     my $descr;
2143     my $section = '';
2144
2145     my @lines = split(/\n/, $raw);
2146     foreach my $line (@lines) {
2147         next if $line =~ m/^\s*$/;
2148
2149         if ($line =~ m/^\[PENDING\]\s*$/i) {
2150             $section = 'pending';
2151             if (defined($descr)) {
2152                 $descr =~ s/\s+$//;
2153                 $conf->{description} = $descr;
2154             }
2155             $descr = undef;
2156             $conf = $res->{$section} = {};
2157             next;
2158
2159         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2160             $section = $1;
2161             if (defined($descr)) {
2162                 $descr =~ s/\s+$//;
2163                 $conf->{description} = $descr;
2164             }
2165             $descr = undef;
2166             $conf = $res->{snapshots}->{$section} = {};
2167             next;
2168         }
2169
2170         if ($line =~ m/^\#(.*)\s*$/) {
2171             $descr = '' if !defined($descr);
2172             $descr .= PVE::Tools::decode_text($1) . "\n";
2173             next;
2174         }
2175
2176         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2177             $descr = '' if !defined($descr);
2178             $descr .= PVE::Tools::decode_text($2);
2179         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2180             $conf->{snapstate} = $1;
2181         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2182             my $key = $1;
2183             my $value = $2;
2184             $conf->{$key} = $value;
2185         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2186             my $value = $1;
2187             if ($section eq 'pending') {
2188                 $conf->{delete} = $value; # we parse this later
2189             } else {
2190                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2191             }
2192         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2193             my $key = $1;
2194             my $value = $2;
2195             eval { $value = check_type($key, $value); };
2196             if ($@) {
2197                 warn "vm $vmid - unable to parse value of '$key' - $@";
2198             } else {
2199                 $key = 'ide2' if $key eq 'cdrom';
2200                 my $fmt = $confdesc->{$key}->{format};
2201                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2202                     my $v = parse_drive($key, $value);
2203                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2204                         $v->{file} = $volid;
2205                         $value = print_drive($v);
2206                     } else {
2207                         warn "vm $vmid - unable to parse value of '$key'\n";
2208                         next;
2209                     }
2210                 }
2211
2212                 $conf->{$key} = $value;
2213             }
2214         }
2215     }
2216
2217     if (defined($descr)) {
2218         $descr =~ s/\s+$//;
2219         $conf->{description} = $descr;
2220     }
2221     delete $res->{snapstate}; # just to be sure
2222
2223     return $res;
2224 }
2225
2226 sub write_vm_config {
2227     my ($filename, $conf) = @_;
2228
2229     delete $conf->{snapstate}; # just to be sure
2230
2231     if ($conf->{cdrom}) {
2232         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2233         $conf->{ide2} = $conf->{cdrom};
2234         delete $conf->{cdrom};
2235     }
2236
2237     # we do not use 'smp' any longer
2238     if ($conf->{sockets}) {
2239         delete $conf->{smp};
2240     } elsif ($conf->{smp}) {
2241         $conf->{sockets} = $conf->{smp};
2242         delete $conf->{cores};
2243         delete $conf->{smp};
2244     }
2245
2246     my $used_volids = {};
2247
2248     my $cleanup_config = sub {
2249         my ($cref, $pending, $snapname) = @_;
2250
2251         foreach my $key (keys %$cref) {
2252             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2253                 $key eq 'snapstate' || $key eq 'pending';
2254             my $value = $cref->{$key};
2255             if ($key eq 'delete') {
2256                 die "propertry 'delete' is only allowed in [PENDING]\n"
2257                     if !$pending;
2258                 # fixme: check syntax?
2259                 next;
2260             }
2261             eval { $value = check_type($key, $value); };
2262             die "unable to parse value of '$key' - $@" if $@;
2263
2264             $cref->{$key} = $value;
2265
2266             if (!$snapname && is_valid_drivename($key)) {
2267                 my $drive = parse_drive($key, $value);
2268                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2269             }
2270         }
2271     };
2272
2273     &$cleanup_config($conf);
2274
2275     &$cleanup_config($conf->{pending}, 1);
2276
2277     foreach my $snapname (keys %{$conf->{snapshots}}) {
2278         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2279         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2280     }
2281
2282     # remove 'unusedX' settings if we re-add a volume
2283     foreach my $key (keys %$conf) {
2284         my $value = $conf->{$key};
2285         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2286             delete $conf->{$key};
2287         }
2288     }
2289
2290     my $generate_raw_config = sub {
2291         my ($conf, $pending) = @_;
2292
2293         my $raw = '';
2294
2295         # add description as comment to top of file
2296         if (defined(my $descr = $conf->{description})) {
2297             if ($descr) {
2298                 foreach my $cl (split(/\n/, $descr)) {
2299                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2300                 }
2301             } else {
2302                 $raw .= "#\n" if $pending;
2303             }
2304         }
2305
2306         foreach my $key (sort keys %$conf) {
2307             next if $key =~ /^(digest|description|pending|snapshots)$/;
2308             $raw .= "$key: $conf->{$key}\n";
2309         }
2310         return $raw;
2311     };
2312
2313     my $raw = &$generate_raw_config($conf);
2314
2315     if (scalar(keys %{$conf->{pending}})){
2316         $raw .= "\n[PENDING]\n";
2317         $raw .= &$generate_raw_config($conf->{pending}, 1);
2318     }
2319
2320     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2321         $raw .= "\n[$snapname]\n";
2322         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2323     }
2324
2325     return $raw;
2326 }
2327
2328 sub load_defaults {
2329
2330     my $res = {};
2331
2332     # we use static defaults from our JSON schema configuration
2333     foreach my $key (keys %$confdesc) {
2334         if (defined(my $default = $confdesc->{$key}->{default})) {
2335             $res->{$key} = $default;
2336         }
2337     }
2338
2339     return $res;
2340 }
2341
2342 sub config_list {
2343     my $vmlist = PVE::Cluster::get_vmlist();
2344     my $res = {};
2345     return $res if !$vmlist || !$vmlist->{ids};
2346     my $ids = $vmlist->{ids};
2347     my $nodename = nodename();
2348
2349     foreach my $vmid (keys %$ids) {
2350         my $d = $ids->{$vmid};
2351         next if !$d->{node} || $d->{node} ne $nodename;
2352         next if !$d->{type} || $d->{type} ne 'qemu';
2353         $res->{$vmid}->{exists} = 1;
2354     }
2355     return $res;
2356 }
2357
2358 # test if VM uses local resources (to prevent migration)
2359 sub check_local_resources {
2360     my ($conf, $noerr) = @_;
2361
2362     my @loc_res = ();
2363
2364     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2365     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2366
2367     push @loc_res, "ivshmem" if $conf->{ivshmem};
2368
2369     foreach my $k (keys %$conf) {
2370         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2371         # sockets are safe: they will recreated be on the target side post-migrate
2372         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2373         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2374     }
2375
2376     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2377
2378     return \@loc_res;
2379 }
2380
2381 # check if used storages are available on all nodes (use by migrate)
2382 sub check_storage_availability {
2383     my ($storecfg, $conf, $node) = @_;
2384
2385     PVE::QemuConfig->foreach_volume($conf, sub {
2386         my ($ds, $drive) = @_;
2387
2388         my $volid = $drive->{file};
2389         return if !$volid;
2390
2391         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2392         return if !$sid;
2393
2394         # check if storage is available on both nodes
2395         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2396         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2397    });
2398 }
2399
2400 # list nodes where all VM images are available (used by has_feature API)
2401 sub shared_nodes {
2402     my ($conf, $storecfg) = @_;
2403
2404     my $nodelist = PVE::Cluster::get_nodelist();
2405     my $nodehash = { map { $_ => 1 } @$nodelist };
2406     my $nodename = nodename();
2407
2408     PVE::QemuConfig->foreach_volume($conf, sub {
2409         my ($ds, $drive) = @_;
2410
2411         my $volid = $drive->{file};
2412         return if !$volid;
2413
2414         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2415         if ($storeid) {
2416             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2417             if ($scfg->{disable}) {
2418                 $nodehash = {};
2419             } elsif (my $avail = $scfg->{nodes}) {
2420                 foreach my $node (keys %$nodehash) {
2421                     delete $nodehash->{$node} if !$avail->{$node};
2422                 }
2423             } elsif (!$scfg->{shared}) {
2424                 foreach my $node (keys %$nodehash) {
2425                     delete $nodehash->{$node} if $node ne $nodename
2426                 }
2427             }
2428         }
2429     });
2430
2431     return $nodehash
2432 }
2433
2434 sub check_local_storage_availability {
2435     my ($conf, $storecfg) = @_;
2436
2437     my $nodelist = PVE::Cluster::get_nodelist();
2438     my $nodehash = { map { $_ => {} } @$nodelist };
2439
2440     PVE::QemuConfig->foreach_volume($conf, sub {
2441         my ($ds, $drive) = @_;
2442
2443         my $volid = $drive->{file};
2444         return if !$volid;
2445
2446         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2447         if ($storeid) {
2448             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2449
2450             if ($scfg->{disable}) {
2451                 foreach my $node (keys %$nodehash) {
2452                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2453                 }
2454             } elsif (my $avail = $scfg->{nodes}) {
2455                 foreach my $node (keys %$nodehash) {
2456                     if (!$avail->{$node}) {
2457                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2458                     }
2459                 }
2460             }
2461         }
2462     });
2463
2464     foreach my $node (values %$nodehash) {
2465         if (my $unavail = $node->{unavailable_storages}) {
2466             $node->{unavailable_storages} = [ sort keys %$unavail ];
2467         }
2468     }
2469
2470     return $nodehash
2471 }
2472
2473 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2474 sub check_running {
2475     my ($vmid, $nocheck, $node) = @_;
2476
2477     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2478     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2479 }
2480
2481 sub vzlist {
2482
2483     my $vzlist = config_list();
2484
2485     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2486
2487     while (defined(my $de = $fd->read)) {
2488         next if $de !~ m/^(\d+)\.pid$/;
2489         my $vmid = $1;
2490         next if !defined($vzlist->{$vmid});
2491         if (my $pid = check_running($vmid)) {
2492             $vzlist->{$vmid}->{pid} = $pid;
2493         }
2494     }
2495
2496     return $vzlist;
2497 }
2498
2499 our $vmstatus_return_properties = {
2500     vmid => get_standard_option('pve-vmid'),
2501     status => {
2502         description => "Qemu process status.",
2503         type => 'string',
2504         enum => ['stopped', 'running'],
2505     },
2506     maxmem => {
2507         description => "Maximum memory in bytes.",
2508         type => 'integer',
2509         optional => 1,
2510         renderer => 'bytes',
2511     },
2512     maxdisk => {
2513         description => "Root disk size in bytes.",
2514         type => 'integer',
2515         optional => 1,
2516         renderer => 'bytes',
2517     },
2518     name => {
2519         description => "VM name.",
2520         type => 'string',
2521         optional => 1,
2522     },
2523     qmpstatus => {
2524         description => "Qemu QMP agent status.",
2525         type => 'string',
2526         optional => 1,
2527     },
2528     pid => {
2529         description => "PID of running qemu process.",
2530         type => 'integer',
2531         optional => 1,
2532     },
2533     uptime => {
2534         description => "Uptime.",
2535         type => 'integer',
2536         optional => 1,
2537         renderer => 'duration',
2538     },
2539     cpus => {
2540         description => "Maximum usable CPUs.",
2541         type => 'number',
2542         optional => 1,
2543     },
2544     lock => {
2545         description => "The current config lock, if any.",
2546         type => 'string',
2547         optional => 1,
2548     },
2549     tags => {
2550         description  => "The current configured tags, if any",
2551         type => 'string',
2552         optional => 1,
2553     },
2554 };
2555
2556 my $last_proc_pid_stat;
2557
2558 # get VM status information
2559 # This must be fast and should not block ($full == false)
2560 # We only query KVM using QMP if $full == true (this can be slow)
2561 sub vmstatus {
2562     my ($opt_vmid, $full) = @_;
2563
2564     my $res = {};
2565
2566     my $storecfg = PVE::Storage::config();
2567
2568     my $list = vzlist();
2569     my $defaults = load_defaults();
2570
2571     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2572
2573     my $cpucount = $cpuinfo->{cpus} || 1;
2574
2575     foreach my $vmid (keys %$list) {
2576         next if $opt_vmid && ($vmid ne $opt_vmid);
2577
2578         my $conf = PVE::QemuConfig->load_config($vmid);
2579
2580         my $d = { vmid => $vmid };
2581         $d->{pid} = $list->{$vmid}->{pid};
2582
2583         # fixme: better status?
2584         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2585
2586         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2587         if (defined($size)) {
2588             $d->{disk} = 0; # no info available
2589             $d->{maxdisk} = $size;
2590         } else {
2591             $d->{disk} = 0;
2592             $d->{maxdisk} = 0;
2593         }
2594
2595         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2596             * ($conf->{cores} || $defaults->{cores});
2597         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2598         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2599
2600         $d->{name} = $conf->{name} || "VM $vmid";
2601         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2602             : $defaults->{memory}*(1024*1024);
2603
2604         if ($conf->{balloon}) {
2605             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2606             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2607                 : $defaults->{shares};
2608         }
2609
2610         $d->{uptime} = 0;
2611         $d->{cpu} = 0;
2612         $d->{mem} = 0;
2613
2614         $d->{netout} = 0;
2615         $d->{netin} = 0;
2616
2617         $d->{diskread} = 0;
2618         $d->{diskwrite} = 0;
2619
2620         $d->{template} = PVE::QemuConfig->is_template($conf);
2621
2622         $d->{serial} = 1 if conf_has_serial($conf);
2623         $d->{lock} = $conf->{lock} if $conf->{lock};
2624         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2625
2626         $res->{$vmid} = $d;
2627     }
2628
2629     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2630     foreach my $dev (keys %$netdev) {
2631         next if $dev !~ m/^tap([1-9]\d*)i/;
2632         my $vmid = $1;
2633         my $d = $res->{$vmid};
2634         next if !$d;
2635
2636         $d->{netout} += $netdev->{$dev}->{receive};
2637         $d->{netin} += $netdev->{$dev}->{transmit};
2638
2639         if ($full) {
2640             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2641             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2642         }
2643
2644     }
2645
2646     my $ctime = gettimeofday;
2647
2648     foreach my $vmid (keys %$list) {
2649
2650         my $d = $res->{$vmid};
2651         my $pid = $d->{pid};
2652         next if !$pid;
2653
2654         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2655         next if !$pstat; # not running
2656
2657         my $used = $pstat->{utime} + $pstat->{stime};
2658
2659         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2660
2661         if ($pstat->{vsize}) {
2662             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2663         }
2664
2665         my $old = $last_proc_pid_stat->{$pid};
2666         if (!$old) {
2667             $last_proc_pid_stat->{$pid} = {
2668                 time => $ctime,
2669                 used => $used,
2670                 cpu => 0,
2671             };
2672             next;
2673         }
2674
2675         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2676
2677         if ($dtime > 1000) {
2678             my $dutime = $used -  $old->{used};
2679
2680             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2681             $last_proc_pid_stat->{$pid} = {
2682                 time => $ctime,
2683                 used => $used,
2684                 cpu => $d->{cpu},
2685             };
2686         } else {
2687             $d->{cpu} = $old->{cpu};
2688         }
2689     }
2690
2691     return $res if !$full;
2692
2693     my $qmpclient = PVE::QMPClient->new();
2694
2695     my $ballooncb = sub {
2696         my ($vmid, $resp) = @_;
2697
2698         my $info = $resp->{'return'};
2699         return if !$info->{max_mem};
2700
2701         my $d = $res->{$vmid};
2702
2703         # use memory assigned to VM
2704         $d->{maxmem} = $info->{max_mem};
2705         $d->{balloon} = $info->{actual};
2706
2707         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2708             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2709             $d->{freemem} = $info->{free_mem};
2710         }
2711
2712         $d->{ballooninfo} = $info;
2713     };
2714
2715     my $blockstatscb = sub {
2716         my ($vmid, $resp) = @_;
2717         my $data = $resp->{'return'} || [];
2718         my $totalrdbytes = 0;
2719         my $totalwrbytes = 0;
2720
2721         for my $blockstat (@$data) {
2722             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2723             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2724
2725             $blockstat->{device} =~ s/drive-//;
2726             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2727         }
2728         $res->{$vmid}->{diskread} = $totalrdbytes;
2729         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2730     };
2731
2732     my $statuscb = sub {
2733         my ($vmid, $resp) = @_;
2734
2735         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2736         # this fails if ballon driver is not loaded, so this must be
2737         # the last commnand (following command are aborted if this fails).
2738         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2739
2740         my $status = 'unknown';
2741         if (!defined($status = $resp->{'return'}->{status})) {
2742             warn "unable to get VM status\n";
2743             return;
2744         }
2745
2746         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2747     };
2748
2749     foreach my $vmid (keys %$list) {
2750         next if $opt_vmid && ($vmid ne $opt_vmid);
2751         next if !$res->{$vmid}->{pid}; # not running
2752         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2753     }
2754
2755     $qmpclient->queue_execute(undef, 2);
2756
2757     foreach my $vmid (keys %$list) {
2758         next if $opt_vmid && ($vmid ne $opt_vmid);
2759         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2760     }
2761
2762     return $res;
2763 }
2764
2765 sub conf_has_serial {
2766     my ($conf) = @_;
2767
2768     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2769         if ($conf->{"serial$i"}) {
2770             return 1;
2771         }
2772     }
2773
2774     return 0;
2775 }
2776
2777 sub conf_has_audio {
2778     my ($conf, $id) = @_;
2779
2780     $id //= 0;
2781     my $audio = $conf->{"audio$id"};
2782     return if !defined($audio);
2783
2784     my $audioproperties = parse_property_string($audio_fmt, $audio);
2785     my $audiodriver = $audioproperties->{driver} // 'spice';
2786
2787     return {
2788         dev => $audioproperties->{device},
2789         dev_id => "audiodev$id",
2790         backend => $audiodriver,
2791         backend_id => "$audiodriver-backend${id}",
2792     };
2793 }
2794
2795 sub audio_devs {
2796     my ($audio, $audiopciaddr, $machine_version) = @_;
2797
2798     my $devs = [];
2799
2800     my $id = $audio->{dev_id};
2801     my $audiodev = "";
2802     if (min_version($machine_version, 4, 2)) {
2803         $audiodev = ",audiodev=$audio->{backend_id}";
2804     }
2805
2806     if ($audio->{dev} eq 'AC97') {
2807         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
2808     } elsif ($audio->{dev} =~ /intel\-hda$/) {
2809         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
2810         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
2811         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
2812     } else {
2813         die "unkown audio device '$audio->{dev}', implement me!";
2814     }
2815
2816     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
2817
2818     return $devs;
2819 }
2820
2821 sub vga_conf_has_spice {
2822     my ($vga) = @_;
2823
2824     my $vgaconf = parse_vga($vga);
2825     my $vgatype = $vgaconf->{type};
2826     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
2827
2828     return $1 || 1;
2829 }
2830
2831 sub is_native($) {
2832     my ($arch) = @_;
2833     return get_host_arch() eq $arch;
2834 }
2835
2836 sub get_vm_arch {
2837     my ($conf) = @_;
2838     return $conf->{arch} // get_host_arch();
2839 }
2840
2841 my $default_machines = {
2842     x86_64 => 'pc',
2843     aarch64 => 'virt',
2844 };
2845
2846 sub get_vm_machine {
2847     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
2848
2849     my $machine = $forcemachine || $conf->{machine};
2850
2851     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
2852         $arch //= 'x86_64';
2853         $machine ||= $default_machines->{$arch};
2854         if ($add_pve_version) {
2855             $kvmversion //= kvm_user_version();
2856             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
2857             $machine .= "+pve$pvever";
2858         }
2859     }
2860
2861     if ($add_pve_version && $machine !~ m/\+pve\d+$/) {
2862         # for version-pinned machines that do not include a pve-version (e.g.
2863         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
2864         $machine .= '+pve0';
2865     }
2866
2867     return $machine;
2868 }
2869
2870 sub get_ovmf_files($) {
2871     my ($arch) = @_;
2872
2873     my $ovmf = $OVMF->{$arch}
2874         or die "no OVMF images known for architecture '$arch'\n";
2875
2876     return @$ovmf;
2877 }
2878
2879 my $Arch2Qemu = {
2880     aarch64 => '/usr/bin/qemu-system-aarch64',
2881     x86_64 => '/usr/bin/qemu-system-x86_64',
2882 };
2883 sub get_command_for_arch($) {
2884     my ($arch) = @_;
2885     return '/usr/bin/kvm' if is_native($arch);
2886
2887     my $cmd = $Arch2Qemu->{$arch}
2888         or die "don't know how to emulate architecture '$arch'\n";
2889     return $cmd;
2890 }
2891
2892 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
2893 # to use in a QEMU command line (-cpu element), first array_intersect the result
2894 # of query_supported_ with query_understood_. This is necessary because:
2895 #
2896 # a) query_understood_ returns flags the host cannot use and
2897 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
2898 #    flags, but CPU settings - with most of them being flags. Those settings
2899 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
2900 #
2901 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
2902 # expensive. If you need the value returned from this, you can get it much
2903 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
2904 # $accel being 'kvm' or 'tcg'.
2905 #
2906 # pvestatd calls this function on startup and whenever the QEMU/KVM version
2907 # changes, automatically populating pmxcfs.
2908 #
2909 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
2910 # since kvm and tcg machines support different flags
2911 #
2912 sub query_supported_cpu_flags {
2913     my ($arch) = @_;
2914
2915     $arch //= get_host_arch();
2916     my $default_machine = $default_machines->{$arch};
2917
2918     my $flags = {};
2919
2920     # FIXME: Once this is merged, the code below should work for ARM as well:
2921     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
2922     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
2923         $arch eq "aarch64";
2924
2925     my $kvm_supported = defined(kvm_version());
2926     my $qemu_cmd = get_command_for_arch($arch);
2927     my $fakevmid = -1;
2928     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
2929
2930     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
2931     my $query_supported_run_qemu = sub {
2932         my ($kvm) = @_;
2933
2934         my $flags = {};
2935         my $cmd = [
2936             $qemu_cmd,
2937             '-machine', $default_machine,
2938             '-display', 'none',
2939             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server,nowait",
2940             '-mon', 'chardev=qmp,mode=control',
2941             '-pidfile', $pidfile,
2942             '-S', '-daemonize'
2943         ];
2944
2945         if (!$kvm) {
2946             push @$cmd, '-accel', 'tcg';
2947         }
2948
2949         my $rc = run_command($cmd, noerr => 1, quiet => 0);
2950         die "QEMU flag querying VM exited with code " . $rc if $rc;
2951
2952         eval {
2953             my $cmd_result = mon_cmd(
2954                 $fakevmid,
2955                 'query-cpu-model-expansion',
2956                 type => 'full',
2957                 model => { name => 'host' }
2958             );
2959
2960             my $props = $cmd_result->{model}->{props};
2961             foreach my $prop (keys %$props) {
2962                 next if $props->{$prop} ne '1';
2963                 # QEMU returns some flags multiple times, with '_', '.' or '-'
2964                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
2965                 # We only keep those with underscores, to match /proc/cpuinfo
2966                 $prop =~ s/\.|-/_/g;
2967                 $flags->{$prop} = 1;
2968             }
2969         };
2970         my $err = $@;
2971
2972         # force stop with 10 sec timeout and 'nocheck'
2973         # always stop, even if QMP failed
2974         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
2975
2976         die $err if $err;
2977
2978         return [ sort keys %$flags ];
2979     };
2980
2981     # We need to query QEMU twice, since KVM and TCG have different supported flags
2982     PVE::QemuConfig->lock_config($fakevmid, sub {
2983         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
2984         warn "warning: failed querying supported tcg flags: $@\n" if $@;
2985
2986         if ($kvm_supported) {
2987             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
2988             warn "warning: failed querying supported kvm flags: $@\n" if $@;
2989         }
2990     });
2991
2992     return $flags;
2993 }
2994
2995 # Understood CPU flags are written to a file at 'pve-qemu' compile time
2996 my $understood_cpu_flag_dir = "/usr/share/kvm";
2997 sub query_understood_cpu_flags {
2998     my $arch = get_host_arch();
2999     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3000
3001     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3002         if ! -e $filepath;
3003
3004     my $raw = file_get_contents($filepath);
3005     $raw =~ s/^\s+|\s+$//g;
3006     my @flags = split(/\s+/, $raw);
3007
3008     return \@flags;
3009 }
3010
3011 sub config_to_command {
3012     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu) = @_;
3013
3014     my $cmd = [];
3015     my $globalFlags = [];
3016     my $machineFlags = [];
3017     my $rtcFlags = [];
3018     my $devices = [];
3019     my $pciaddr = '';
3020     my $bridges = {};
3021     my $ostype = $conf->{ostype};
3022     my $winversion = windows_version($ostype);
3023     my $kvm = $conf->{kvm};
3024     my $nodename = nodename();
3025
3026     my $arch = get_vm_arch($conf);
3027     my $kvm_binary = get_command_for_arch($arch);
3028     my $kvmver = kvm_user_version($kvm_binary);
3029
3030     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3031         $kvmver //= "undefined";
3032         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3033     }
3034
3035     my $add_pve_version = min_version($kvmver, 4, 1);
3036
3037     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3038     my $machine_version = extract_version($machine_type, $kvmver);
3039     $kvm //= 1 if is_native($arch);
3040
3041     $machine_version =~ m/(\d+)\.(\d+)/;
3042     my ($machine_major, $machine_minor) = ($1, $2);
3043
3044     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3045         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3046     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3047         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3048             ." please upgrade node '$nodename'\n"
3049     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3050         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3051         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3052             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3053             ." node '$nodename'\n";
3054     }
3055
3056     # if a specific +pve version is required for a feature, use $version_guard
3057     # instead of min_version to allow machines to be run with the minimum
3058     # required version
3059     my $required_pve_version = 0;
3060     my $version_guard = sub {
3061         my ($major, $minor, $pve) = @_;
3062         return 0 if !min_version($machine_version, $major, $minor, $pve);
3063         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3064         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3065         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3066         return 1;
3067     };
3068
3069     if ($kvm && !defined kvm_version()) {
3070         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3071             ." or enable in BIOS.\n";
3072     }
3073
3074     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3075     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3076     my $use_old_bios_files = undef;
3077     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3078
3079     my $cpuunits = defined($conf->{cpuunits}) ?
3080             $conf->{cpuunits} : $defaults->{cpuunits};
3081
3082     push @$cmd, $kvm_binary;
3083
3084     push @$cmd, '-id', $vmid;
3085
3086     my $vmname = $conf->{name} || "vm$vmid";
3087
3088     push @$cmd, '-name', $vmname;
3089
3090     push @$cmd, '-no-shutdown';
3091
3092     my $use_virtio = 0;
3093
3094     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3095     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3096     push @$cmd, '-mon', "chardev=qmp,mode=control";
3097
3098     if (min_version($machine_version, 2, 12)) {
3099         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3100         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3101     }
3102
3103     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3104
3105     push @$cmd, '-daemonize';
3106
3107     if ($conf->{smbios1}) {
3108         my $smbios_conf = parse_smbios1($conf->{smbios1});
3109         if ($smbios_conf->{base64}) {
3110             # Do not pass base64 flag to qemu
3111             delete $smbios_conf->{base64};
3112             my $smbios_string = "";
3113             foreach my $key (keys %$smbios_conf) {
3114                 my $value;
3115                 if ($key eq "uuid") {
3116                     $value = $smbios_conf->{uuid}
3117                 } else {
3118                     $value = decode_base64($smbios_conf->{$key});
3119                 }
3120                 # qemu accepts any binary data, only commas need escaping by double comma
3121                 $value =~ s/,/,,/g;
3122                 $smbios_string .= "," . $key . "=" . $value if $value;
3123             }
3124             push @$cmd, '-smbios', "type=1" . $smbios_string;
3125         } else {
3126             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3127         }
3128     }
3129
3130     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3131         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3132         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3133
3134         my ($path, $format);
3135         if (my $efidisk = $conf->{efidisk0}) {
3136             my $d = parse_drive('efidisk0', $efidisk);
3137             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3138             $format = $d->{format};
3139             if ($storeid) {
3140                 $path = PVE::Storage::path($storecfg, $d->{file});
3141                 if (!defined($format)) {
3142                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3143                     $format = qemu_img_format($scfg, $volname);
3144                 }
3145             } else {
3146                 $path = $d->{file};
3147                 die "efidisk format must be specified\n"
3148                     if !defined($format);
3149             }
3150         } else {
3151             warn "no efidisk configured! Using temporary efivars disk.\n";
3152             $path = "/tmp/$vmid-ovmf.fd";
3153             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3154             $format = 'raw';
3155         }
3156
3157         my $size_str = "";
3158
3159         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3160             $size_str = ",size=" . (-s $ovmf_vars);
3161         }
3162
3163         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3164         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0$size_str,file=$path";
3165     }
3166
3167     # load q35 config
3168     if ($q35) {
3169         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3170         if (min_version($machine_version, 4, 0)) {
3171             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3172         } else {
3173             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3174         }
3175     }
3176
3177     if ($conf->{vmgenid}) {
3178         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3179     }
3180
3181     # add usb controllers
3182     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3183         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3184     push @$devices, @usbcontrollers if @usbcontrollers;
3185     my $vga = parse_vga($conf->{vga});
3186
3187     my $qxlnum = vga_conf_has_spice($conf->{vga});
3188     $vga->{type} = 'qxl' if $qxlnum;
3189
3190     if (!$vga->{type}) {
3191         if ($arch eq 'aarch64') {
3192             $vga->{type} = 'virtio';
3193         } elsif (min_version($machine_version, 2, 9)) {
3194             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3195         } else {
3196             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3197         }
3198     }
3199
3200     # enable absolute mouse coordinates (needed by vnc)
3201     my $tablet;
3202     if (defined($conf->{tablet})) {
3203         $tablet = $conf->{tablet};
3204     } else {
3205         $tablet = $defaults->{tablet};
3206         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3207         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3208     }
3209
3210     if ($tablet) {
3211         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3212         my $kbd = print_keyboarddevice_full($conf, $arch);
3213         push @$devices, '-device', $kbd if defined($kbd);
3214     }
3215
3216     my $bootorder = device_bootorder($conf);
3217
3218     # host pci device passthrough
3219     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3220         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3221
3222     # usb devices
3223     my $usb_dev_features = {};
3224     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3225
3226     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3227         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3228     push @$devices, @usbdevices if @usbdevices;
3229
3230     # serial devices
3231     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3232         if (my $path = $conf->{"serial$i"}) {
3233             if ($path eq 'socket') {
3234                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3235                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3236                 # On aarch64, serial0 is the UART device. Qemu only allows
3237                 # connecting UART devices via the '-serial' command line, as
3238                 # the device has a fixed slot on the hardware...
3239                 if ($arch eq 'aarch64' && $i == 0) {
3240                     push @$devices, '-serial', "chardev:serial$i";
3241                 } else {
3242                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3243                 }
3244             } else {
3245                 die "no such serial device\n" if ! -c $path;
3246                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3247                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3248             }
3249         }
3250     }
3251
3252     # parallel devices
3253     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3254         if (my $path = $conf->{"parallel$i"}) {
3255             die "no such parallel device\n" if ! -c $path;
3256             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3257             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3258             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3259         }
3260     }
3261
3262     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3263         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3264         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3265         push @$devices, @$audio_devs;
3266     }
3267
3268     my $sockets = 1;
3269     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3270     $sockets = $conf->{sockets} if  $conf->{sockets};
3271
3272     my $cores = $conf->{cores} || 1;
3273
3274     my $maxcpus = $sockets * $cores;
3275
3276     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3277
3278     my $allowed_vcpus = $cpuinfo->{cpus};
3279
3280     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3281         if ($allowed_vcpus < $maxcpus);
3282
3283     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3284
3285         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3286         for (my $i = 2; $i <= $vcpus; $i++)  {
3287             my $cpustr = print_cpu_device($conf,$i);
3288             push @$cmd, '-device', $cpustr;
3289         }
3290
3291     } else {
3292
3293         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3294     }
3295     push @$cmd, '-nodefaults';
3296
3297     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3298
3299     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3300
3301     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3302
3303     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3304         push @$devices, '-device', print_vga_device(
3305             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3306         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3307         push @$cmd,  '-vnc', "unix:$socket,password";
3308     } else {
3309         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3310         push @$cmd, '-nographic';
3311     }
3312
3313     # time drift fix
3314     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3315     my $useLocaltime = $conf->{localtime};
3316
3317     if ($winversion >= 5) { # windows
3318         $useLocaltime = 1 if !defined($conf->{localtime});
3319
3320         # use time drift fix when acpi is enabled
3321         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3322             $tdf = 1 if !defined($conf->{tdf});
3323         }
3324     }
3325
3326     if ($winversion >= 6) {
3327         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3328         push @$cmd, '-no-hpet';
3329     }
3330
3331     push @$rtcFlags, 'driftfix=slew' if $tdf;
3332
3333     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3334         push @$rtcFlags, "base=$conf->{startdate}";
3335     } elsif ($useLocaltime) {
3336         push @$rtcFlags, 'base=localtime';
3337     }
3338
3339     if ($forcecpu) {
3340         push @$cmd, '-cpu', $forcecpu;
3341     } else {
3342         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3343     }
3344
3345     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3346
3347     push @$cmd, '-S' if $conf->{freeze};
3348
3349     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3350
3351     my $guest_agent = parse_guest_agent($conf);
3352
3353     if ($guest_agent->{enabled}) {
3354         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3355         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3356
3357         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3358             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3359             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3360             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3361         } elsif ($guest_agent->{type} eq 'isa') {
3362             push @$devices, '-device', "isa-serial,chardev=qga0";
3363         }
3364     }
3365
3366     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3367     if ($rng && $version_guard->(4, 1, 2)) {
3368         check_rng_source($rng->{source});
3369
3370         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3371         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3372         my $limiter_str = "";
3373         if ($max_bytes) {
3374             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3375         }
3376
3377         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3378         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3379         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3380     }
3381
3382     my $spice_port;
3383
3384     if ($qxlnum) {
3385         if ($qxlnum > 1) {
3386             if ($winversion){
3387                 for (my $i = 1; $i < $qxlnum; $i++){
3388                     push @$devices, '-device', print_vga_device(
3389                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3390                 }
3391             } else {
3392                 # assume other OS works like Linux
3393                 my ($ram, $vram) = ("134217728", "67108864");
3394                 if ($vga->{memory}) {
3395                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3396                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3397                 }
3398                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3399                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3400             }
3401         }
3402
3403         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3404
3405         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3406         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3407         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3408
3409         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3410         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3411         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3412
3413         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3414         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3415
3416         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3417         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3418         if ($spice_enhancement->{foldersharing}) {
3419             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3420             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3421         }
3422
3423         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3424         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3425             if $spice_enhancement->{videostreaming};
3426
3427         push @$devices, '-spice', "$spice_opts";
3428     }
3429
3430     # enable balloon by default, unless explicitly disabled
3431     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3432         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3433         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3434     }
3435
3436     if ($conf->{watchdog}) {
3437         my $wdopts = parse_watchdog($conf->{watchdog});
3438         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3439         my $watchdog = $wdopts->{model} || 'i6300esb';
3440         push @$devices, '-device', "$watchdog$pciaddr";
3441         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3442     }
3443
3444     my $vollist = [];
3445     my $scsicontroller = {};
3446     my $ahcicontroller = {};
3447     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3448
3449     # Add iscsi initiator name if available
3450     if (my $initiator = get_initiator_name()) {
3451         push @$devices, '-iscsi', "initiator-name=$initiator";
3452     }
3453
3454     PVE::QemuConfig->foreach_volume($conf, sub {
3455         my ($ds, $drive) = @_;
3456
3457         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3458             push @$vollist, $drive->{file};
3459         }
3460
3461         # ignore efidisk here, already added in bios/fw handling code above
3462         return if $drive->{interface} eq 'efidisk';
3463
3464         $use_virtio = 1 if $ds =~ m/^virtio/;
3465
3466         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3467
3468         if ($drive->{interface} eq 'virtio'){
3469            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3470         }
3471
3472         if ($drive->{interface} eq 'scsi') {
3473
3474             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3475
3476             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3477                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3478
3479             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3480             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3481
3482             my $iothread = '';
3483             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3484                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3485                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3486             } elsif ($drive->{iothread}) {
3487                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3488             }
3489
3490             my $queues = '';
3491             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3492                 $queues = ",num_queues=$drive->{queues}";
3493             }
3494
3495             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3496                 if !$scsicontroller->{$controller};
3497             $scsicontroller->{$controller}=1;
3498         }
3499
3500         if ($drive->{interface} eq 'sata') {
3501             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3502             $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3503             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3504                 if !$ahcicontroller->{$controller};
3505             $ahcicontroller->{$controller}=1;
3506         }
3507
3508         my $drive_cmd = print_drive_commandline_full($storecfg, $vmid, $drive);
3509         $drive_cmd .= ',readonly' if PVE::QemuConfig->is_template($conf);
3510
3511         push @$devices, '-drive',$drive_cmd;
3512         push @$devices, '-device', print_drivedevice_full(
3513             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3514     });
3515
3516     for (my $i = 0; $i < $MAX_NETS; $i++) {
3517         my $netname = "net$i";
3518
3519         next if !$conf->{$netname};
3520         my $d = parse_net($conf->{$netname});
3521         next if !$d;
3522
3523         $use_virtio = 1 if $d->{model} eq 'virtio';
3524
3525         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
3526
3527         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
3528         push @$devices, '-netdev', $netdevfull;
3529
3530         my $netdevicefull = print_netdevice_full(
3531             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
3532
3533         push @$devices, '-device', $netdevicefull;
3534     }
3535
3536     if ($conf->{ivshmem}) {
3537         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3538
3539         my $bus;
3540         if ($q35) {
3541             $bus = print_pcie_addr("ivshmem");
3542         } else {
3543             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3544         }
3545
3546         my $ivshmem_name = $ivshmem->{name} // $vmid;
3547         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3548
3549         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3550         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
3551             .",size=$ivshmem->{size}M";
3552     }
3553
3554     # pci.4 is nested in pci.1
3555     $bridges->{1} = 1 if $bridges->{4};
3556
3557     if (!$q35) {
3558         # add pci bridges
3559         if (min_version($machine_version, 2, 3)) {
3560            $bridges->{1} = 1;
3561            $bridges->{2} = 1;
3562         }
3563
3564         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3565
3566     }
3567
3568     for my $k (sort {$b cmp $a} keys %$bridges) {
3569         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3570
3571         my $k_name = $k;
3572         if ($k == 2 && $legacy_igd) {
3573             $k_name = "$k-igd";
3574         }
3575         $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
3576
3577         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3578         if ($q35) {
3579             # add after -readconfig pve-q35.cfg
3580             splice @$devices, 2, 0, '-device', $devstr;
3581         } else {
3582             unshift @$devices, '-device', $devstr if $k > 0;
3583         }
3584     }
3585
3586     if (!$kvm) {
3587         push @$machineFlags, 'accel=tcg';
3588     }
3589
3590     my $machine_type_min = $machine_type;
3591     if ($add_pve_version) {
3592         $machine_type_min =~ s/\+pve\d+$//;
3593         $machine_type_min .= "+pve$required_pve_version";
3594     }
3595     push @$machineFlags, "type=${machine_type_min}";
3596
3597     push @$cmd, @$devices;
3598     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
3599     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
3600     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
3601
3602     if (my $vmstate = $conf->{vmstate}) {
3603         my $statepath = PVE::Storage::path($storecfg, $vmstate);
3604         push @$vollist, $vmstate;
3605         push @$cmd, '-loadstate', $statepath;
3606         print "activating and using '$vmstate' as vmstate\n";
3607     }
3608
3609     # add custom args
3610     if ($conf->{args}) {
3611         my $aa = PVE::Tools::split_args($conf->{args});
3612         push @$cmd, @$aa;
3613     }
3614
3615     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3616 }
3617
3618 sub check_rng_source {
3619     my ($source) = @_;
3620
3621     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3622     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
3623         if ! -e $source;
3624
3625     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
3626     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
3627         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
3628         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
3629         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
3630             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
3631             ." to the host.\n";
3632     }
3633 }
3634
3635 sub spice_port {
3636     my ($vmid) = @_;
3637
3638     my $res = mon_cmd($vmid, 'query-spice');
3639
3640     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3641 }
3642
3643 sub vm_devices_list {
3644     my ($vmid) = @_;
3645
3646     my $res = mon_cmd($vmid, 'query-pci');
3647     my $devices_to_check = [];
3648     my $devices = {};
3649     foreach my $pcibus (@$res) {
3650         push @$devices_to_check, @{$pcibus->{devices}},
3651     }
3652
3653     while (@$devices_to_check) {
3654         my $to_check = [];
3655         for my $d (@$devices_to_check) {
3656             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3657             next if !$d->{'pci_bridge'};
3658
3659             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3660             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3661         }
3662         $devices_to_check = $to_check;
3663     }
3664
3665     my $resblock = mon_cmd($vmid, 'query-block');
3666     foreach my $block (@$resblock) {
3667         if($block->{device} =~ m/^drive-(\S+)/){
3668                 $devices->{$1} = 1;
3669         }
3670     }
3671
3672     my $resmice = mon_cmd($vmid, 'query-mice');
3673     foreach my $mice (@$resmice) {
3674         if ($mice->{name} eq 'QEMU HID Tablet') {
3675             $devices->{tablet} = 1;
3676             last;
3677         }
3678     }
3679
3680     # for usb devices there is no query-usb
3681     # but we can iterate over the entries in
3682     # qom-list path=/machine/peripheral
3683     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3684     foreach my $per (@$resperipheral) {
3685         if ($per->{name} =~ m/^usb\d+$/) {
3686             $devices->{$per->{name}} = 1;
3687         }
3688     }
3689
3690     return $devices;
3691 }
3692
3693 sub vm_deviceplug {
3694     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3695
3696     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3697
3698     my $devices_list = vm_devices_list($vmid);
3699     return 1 if defined($devices_list->{$deviceid});
3700
3701     # add PCI bridge if we need it for the device
3702     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
3703
3704     if ($deviceid eq 'tablet') {
3705
3706         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
3707
3708     } elsif ($deviceid eq 'keyboard') {
3709
3710         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
3711
3712     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3713
3714         die "usb hotplug currently not reliable\n";
3715         # since we can't reliably hot unplug all added usb devices and usb
3716         # passthrough breaks live migration we disable usb hotplugging for now
3717         #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3718
3719     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3720
3721         qemu_iothread_add($vmid, $deviceid, $device);
3722
3723         qemu_driveadd($storecfg, $vmid, $device);
3724         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3725
3726         qemu_deviceadd($vmid, $devicefull);
3727         eval { qemu_deviceaddverify($vmid, $deviceid); };
3728         if (my $err = $@) {
3729             eval { qemu_drivedel($vmid, $deviceid); };
3730             warn $@ if $@;
3731             die $err;
3732         }
3733
3734     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3735
3736
3737         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3738         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3739         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3740
3741         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3742
3743         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3744             qemu_iothread_add($vmid, $deviceid, $device);
3745             $devicefull .= ",iothread=iothread-$deviceid";
3746         }
3747
3748         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3749             $devicefull .= ",num_queues=$device->{queues}";
3750         }
3751
3752         qemu_deviceadd($vmid, $devicefull);
3753         qemu_deviceaddverify($vmid, $deviceid);
3754
3755     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3756
3757         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
3758         qemu_driveadd($storecfg, $vmid, $device);
3759
3760         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3761         eval { qemu_deviceadd($vmid, $devicefull); };
3762         if (my $err = $@) {
3763             eval { qemu_drivedel($vmid, $deviceid); };
3764             warn $@ if $@;
3765             die $err;
3766         }
3767
3768     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3769
3770         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
3771
3772         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
3773         my $use_old_bios_files = undef;
3774         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3775
3776         my $netdevicefull = print_netdevice_full(
3777             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
3778         qemu_deviceadd($vmid, $netdevicefull);
3779         eval {
3780             qemu_deviceaddverify($vmid, $deviceid);
3781             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
3782         };
3783         if (my $err = $@) {
3784             eval { qemu_netdevdel($vmid, $deviceid); };
3785             warn $@ if $@;
3786             die $err;
3787         }
3788
3789     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3790
3791         my $bridgeid = $2;
3792         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3793         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3794
3795         qemu_deviceadd($vmid, $devicefull);
3796         qemu_deviceaddverify($vmid, $deviceid);
3797
3798     } else {
3799         die "can't hotplug device '$deviceid'\n";
3800     }
3801
3802     return 1;
3803 }
3804
3805 # fixme: this should raise exceptions on error!
3806 sub vm_deviceunplug {
3807     my ($vmid, $conf, $deviceid) = @_;
3808
3809     my $devices_list = vm_devices_list($vmid);
3810     return 1 if !defined($devices_list->{$deviceid});
3811
3812     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
3813     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
3814
3815     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
3816
3817         qemu_devicedel($vmid, $deviceid);
3818
3819     } elsif ($deviceid =~ m/^usb\d+$/) {
3820
3821         die "usb hotplug currently not reliable\n";
3822         # when unplugging usb devices this way, there may be remaining usb
3823         # controllers/hubs so we disable it for now
3824         #qemu_devicedel($vmid, $deviceid);
3825         #qemu_devicedelverify($vmid, $deviceid);
3826
3827     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3828
3829         qemu_devicedel($vmid, $deviceid);
3830         qemu_devicedelverify($vmid, $deviceid);
3831         qemu_drivedel($vmid, $deviceid);
3832         qemu_iothread_del($conf, $vmid, $deviceid);
3833
3834     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3835
3836         qemu_devicedel($vmid, $deviceid);
3837         qemu_devicedelverify($vmid, $deviceid);
3838         qemu_iothread_del($conf, $vmid, $deviceid);
3839
3840     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3841
3842         qemu_devicedel($vmid, $deviceid);
3843         qemu_drivedel($vmid, $deviceid);
3844         qemu_deletescsihw($conf, $vmid, $deviceid);
3845
3846     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3847
3848         qemu_devicedel($vmid, $deviceid);
3849         qemu_devicedelverify($vmid, $deviceid);
3850         qemu_netdevdel($vmid, $deviceid);
3851
3852     } else {
3853         die "can't unplug device '$deviceid'\n";
3854     }
3855
3856     return 1;
3857 }
3858
3859 sub qemu_deviceadd {
3860     my ($vmid, $devicefull) = @_;
3861
3862     $devicefull = "driver=".$devicefull;
3863     my %options =  split(/[=,]/, $devicefull);
3864
3865     mon_cmd($vmid, "device_add" , %options);
3866 }
3867
3868 sub qemu_devicedel {
3869     my ($vmid, $deviceid) = @_;
3870
3871     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
3872 }
3873
3874 sub qemu_iothread_add {
3875     my($vmid, $deviceid, $device) = @_;
3876
3877     if ($device->{iothread}) {
3878         my $iothreads = vm_iothreads_list($vmid);
3879         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3880     }
3881 }
3882
3883 sub qemu_iothread_del {
3884     my($conf, $vmid, $deviceid) = @_;
3885
3886     my $confid = $deviceid;
3887     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
3888         $confid = 'scsi' . $1;
3889     }
3890     my $device = parse_drive($confid, $conf->{$confid});
3891     if ($device->{iothread}) {
3892         my $iothreads = vm_iothreads_list($vmid);
3893         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3894     }
3895 }
3896
3897 sub qemu_objectadd {
3898     my($vmid, $objectid, $qomtype) = @_;
3899
3900     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3901
3902     return 1;
3903 }
3904
3905 sub qemu_objectdel {
3906     my($vmid, $objectid) = @_;
3907
3908     mon_cmd($vmid, "object-del", id => $objectid);
3909
3910     return 1;
3911 }
3912
3913 sub qemu_driveadd {
3914     my ($storecfg, $vmid, $device) = @_;
3915
3916     my $drive = print_drive_commandline_full($storecfg, $vmid, $device);
3917     $drive =~ s/\\/\\\\/g;
3918     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
3919
3920     # If the command succeeds qemu prints: "OK"
3921     return 1 if $ret =~ m/OK/s;
3922
3923     die "adding drive failed: $ret\n";
3924 }
3925
3926 sub qemu_drivedel {
3927     my($vmid, $deviceid) = @_;
3928
3929     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
3930     $ret =~ s/^\s+//;
3931
3932     return 1 if $ret eq "";
3933
3934     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3935     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3936
3937     die "deleting drive $deviceid failed : $ret\n";
3938 }
3939
3940 sub qemu_deviceaddverify {
3941     my ($vmid, $deviceid) = @_;
3942
3943     for (my $i = 0; $i <= 5; $i++) {
3944          my $devices_list = vm_devices_list($vmid);
3945          return 1 if defined($devices_list->{$deviceid});
3946          sleep 1;
3947     }
3948
3949     die "error on hotplug device '$deviceid'\n";
3950 }
3951
3952
3953 sub qemu_devicedelverify {
3954     my ($vmid, $deviceid) = @_;
3955
3956     # need to verify that the device is correctly removed as device_del
3957     # is async and empty return is not reliable
3958
3959     for (my $i = 0; $i <= 5; $i++) {
3960          my $devices_list = vm_devices_list($vmid);
3961          return 1 if !defined($devices_list->{$deviceid});
3962          sleep 1;
3963     }
3964
3965     die "error on hot-unplugging device '$deviceid'\n";
3966 }
3967
3968 sub qemu_findorcreatescsihw {
3969     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
3970
3971     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3972
3973     my $scsihwid="$controller_prefix$controller";
3974     my $devices_list = vm_devices_list($vmid);
3975
3976     if(!defined($devices_list->{$scsihwid})) {
3977         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
3978     }
3979
3980     return 1;
3981 }
3982
3983 sub qemu_deletescsihw {
3984     my ($conf, $vmid, $opt) = @_;
3985
3986     my $device = parse_drive($opt, $conf->{$opt});
3987
3988     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3989         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3990         return 1;
3991     }
3992
3993     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3994
3995     my $devices_list = vm_devices_list($vmid);
3996     foreach my $opt (keys %{$devices_list}) {
3997         if (is_valid_drivename($opt)) {
3998             my $drive = parse_drive($opt, $conf->{$opt});
3999             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4000                 return 1;
4001             }
4002         }
4003     }
4004
4005     my $scsihwid="scsihw$controller";
4006
4007     vm_deviceunplug($vmid, $conf, $scsihwid);
4008
4009     return 1;
4010 }
4011
4012 sub qemu_add_pci_bridge {
4013     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4014
4015     my $bridges = {};
4016
4017     my $bridgeid;
4018
4019     print_pci_addr($device, $bridges, $arch, $machine_type);
4020
4021     while (my ($k, $v) = each %$bridges) {
4022         $bridgeid = $k;
4023     }
4024     return 1 if !defined($bridgeid) || $bridgeid < 1;
4025
4026     my $bridge = "pci.$bridgeid";
4027     my $devices_list = vm_devices_list($vmid);
4028
4029     if (!defined($devices_list->{$bridge})) {
4030         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4031     }
4032
4033     return 1;
4034 }
4035
4036 sub qemu_set_link_status {
4037     my ($vmid, $device, $up) = @_;
4038
4039     mon_cmd($vmid, "set_link", name => $device,
4040                up => $up ? JSON::true : JSON::false);
4041 }
4042
4043 sub qemu_netdevadd {
4044     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4045
4046     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4047     my %options =  split(/[=,]/, $netdev);
4048
4049     if (defined(my $vhost = $options{vhost})) {
4050         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4051     }
4052
4053     if (defined(my $queues = $options{queues})) {
4054         $options{queues} = $queues + 0;
4055     }
4056
4057     mon_cmd($vmid, "netdev_add",  %options);
4058     return 1;
4059 }
4060
4061 sub qemu_netdevdel {
4062     my ($vmid, $deviceid) = @_;
4063
4064     mon_cmd($vmid, "netdev_del", id => $deviceid);
4065 }
4066
4067 sub qemu_usb_hotplug {
4068     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4069
4070     return if !$device;
4071
4072     # remove the old one first
4073     vm_deviceunplug($vmid, $conf, $deviceid);
4074
4075     # check if xhci controller is necessary and available
4076     if ($device->{usb3}) {
4077
4078         my $devicelist = vm_devices_list($vmid);
4079
4080         if (!$devicelist->{xhci}) {
4081             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4082             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4083         }
4084     }
4085     my $d = parse_usb_device($device->{host});
4086     $d->{usb3} = $device->{usb3};
4087
4088     # add the new one
4089     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4090 }
4091
4092 sub qemu_cpu_hotplug {
4093     my ($vmid, $conf, $vcpus) = @_;
4094
4095     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4096
4097     my $sockets = 1;
4098     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4099     $sockets = $conf->{sockets} if  $conf->{sockets};
4100     my $cores = $conf->{cores} || 1;
4101     my $maxcpus = $sockets * $cores;
4102
4103     $vcpus = $maxcpus if !$vcpus;
4104
4105     die "you can't add more vcpus than maxcpus\n"
4106         if $vcpus > $maxcpus;
4107
4108     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4109
4110     if ($vcpus < $currentvcpus) {
4111
4112         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4113
4114             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4115                 qemu_devicedel($vmid, "cpu$i");
4116                 my $retry = 0;
4117                 my $currentrunningvcpus = undef;
4118                 while (1) {
4119                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4120                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4121                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4122                     $retry++;
4123                     sleep 1;
4124                 }
4125                 #update conf after each succesfull cpu unplug
4126                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4127                 PVE::QemuConfig->write_config($vmid, $conf);
4128             }
4129         } else {
4130             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4131         }
4132
4133         return;
4134     }
4135
4136     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4137     die "vcpus in running vm does not match its configuration\n"
4138         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4139
4140     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4141
4142         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4143             my $cpustr = print_cpu_device($conf, $i);
4144             qemu_deviceadd($vmid, $cpustr);
4145
4146             my $retry = 0;
4147             my $currentrunningvcpus = undef;
4148             while (1) {
4149                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4150                 last if scalar(@{$currentrunningvcpus}) == $i;
4151                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4152                 sleep 1;
4153                 $retry++;
4154             }
4155             #update conf after each succesfull cpu hotplug
4156             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4157             PVE::QemuConfig->write_config($vmid, $conf);
4158         }
4159     } else {
4160
4161         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4162             mon_cmd($vmid, "cpu-add", id => int($i));
4163         }
4164     }
4165 }
4166
4167 sub qemu_block_set_io_throttle {
4168     my ($vmid, $deviceid,
4169         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4170         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4171         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4172         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4173
4174     return if !check_running($vmid) ;
4175
4176     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4177         bps => int($bps),
4178         bps_rd => int($bps_rd),
4179         bps_wr => int($bps_wr),
4180         iops => int($iops),
4181         iops_rd => int($iops_rd),
4182         iops_wr => int($iops_wr),
4183         bps_max => int($bps_max),
4184         bps_rd_max => int($bps_rd_max),
4185         bps_wr_max => int($bps_wr_max),
4186         iops_max => int($iops_max),
4187         iops_rd_max => int($iops_rd_max),
4188         iops_wr_max => int($iops_wr_max),
4189         bps_max_length => int($bps_max_length),
4190         bps_rd_max_length => int($bps_rd_max_length),
4191         bps_wr_max_length => int($bps_wr_max_length),
4192         iops_max_length => int($iops_max_length),
4193         iops_rd_max_length => int($iops_rd_max_length),
4194         iops_wr_max_length => int($iops_wr_max_length),
4195     );
4196
4197 }
4198
4199 sub qemu_block_resize {
4200     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4201
4202     my $running = check_running($vmid);
4203
4204     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4205
4206     return if !$running;
4207
4208     my $padding = (1024 - $size % 1024) % 1024;
4209     $size = $size + $padding;
4210
4211     mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4212
4213 }
4214
4215 sub qemu_volume_snapshot {
4216     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4217
4218     my $running = check_running($vmid);
4219
4220     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4221         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4222     } else {
4223         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4224     }
4225 }
4226
4227 sub qemu_volume_snapshot_delete {
4228     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4229
4230     my $running = check_running($vmid);
4231
4232     if($running) {
4233
4234         $running = undef;
4235         my $conf = PVE::QemuConfig->load_config($vmid);
4236         PVE::QemuConfig->foreach_volume($conf, sub {
4237             my ($ds, $drive) = @_;
4238             $running = 1 if $drive->{file} eq $volid;
4239         });
4240     }
4241
4242     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4243         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4244     } else {
4245         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4246     }
4247 }
4248
4249 sub set_migration_caps {
4250     my ($vmid) = @_;
4251
4252     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4253
4254     my $cap_ref = [];
4255
4256     my $enabled_cap = {
4257         "auto-converge" => 1,
4258         "xbzrle" => 1,
4259         "x-rdma-pin-all" => 0,
4260         "zero-blocks" => 0,
4261         "compress" => 0,
4262         "dirty-bitmaps" => $qemu_support->{'pbs-dirty-bitmap-migration'} ? 1 : 0,
4263     };
4264
4265     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4266
4267     for my $supported_capability (@$supported_capabilities) {
4268         push @$cap_ref, {
4269             capability => $supported_capability->{capability},
4270             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4271         };
4272     }
4273
4274     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4275 }
4276
4277 sub foreach_volid {
4278     my ($conf, $func, @param) = @_;
4279
4280     my $volhash = {};
4281
4282     my $test_volid = sub {
4283         my ($key, $drive, $snapname) = @_;
4284
4285         my $volid = $drive->{file};
4286         return if !$volid;
4287
4288         $volhash->{$volid}->{cdrom} //= 1;
4289         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4290
4291         my $replicate = $drive->{replicate} // 1;
4292         $volhash->{$volid}->{replicate} //= 0;
4293         $volhash->{$volid}->{replicate} = 1 if $replicate;
4294
4295         $volhash->{$volid}->{shared} //= 0;
4296         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4297
4298         $volhash->{$volid}->{referenced_in_config} //= 0;
4299         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4300
4301         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4302             if defined($snapname);
4303
4304         my $size = $drive->{size};
4305         $volhash->{$volid}->{size} //= $size if $size;
4306
4307         $volhash->{$volid}->{is_vmstate} //= 0;
4308         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4309
4310         $volhash->{$volid}->{is_unused} //= 0;
4311         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4312     };
4313
4314     my $include_opts = {
4315         extra_keys => ['vmstate'],
4316         include_unused => 1,
4317     };
4318
4319     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4320     foreach my $snapname (keys %{$conf->{snapshots}}) {
4321         my $snap = $conf->{snapshots}->{$snapname};
4322         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4323     }
4324
4325     foreach my $volid (keys %$volhash) {
4326         &$func($volid, $volhash->{$volid}, @param);
4327     }
4328 }
4329
4330 my $fast_plug_option = {
4331     'lock' => 1,
4332     'name' => 1,
4333     'onboot' => 1,
4334     'shares' => 1,
4335     'startup' => 1,
4336     'description' => 1,
4337     'protection' => 1,
4338     'vmstatestorage' => 1,
4339     'hookscript' => 1,
4340     'tags' => 1,
4341 };
4342
4343 # hotplug changes in [PENDING]
4344 # $selection hash can be used to only apply specified options, for
4345 # example: { cores => 1 } (only apply changed 'cores')
4346 # $errors ref is used to return error messages
4347 sub vmconfig_hotplug_pending {
4348     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4349
4350     my $defaults = load_defaults();
4351     my $arch = get_vm_arch($conf);
4352     my $machine_type = get_vm_machine($conf, undef, $arch);
4353
4354     # commit values which do not have any impact on running VM first
4355     # Note: those option cannot raise errors, we we do not care about
4356     # $selection and always apply them.
4357
4358     my $add_error = sub {
4359         my ($opt, $msg) = @_;
4360         $errors->{$opt} = "hotplug problem - $msg";
4361     };
4362
4363     my $changes = 0;
4364     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4365         if ($fast_plug_option->{$opt}) {
4366             $conf->{$opt} = $conf->{pending}->{$opt};
4367             delete $conf->{pending}->{$opt};
4368             $changes = 1;
4369         }
4370     }
4371
4372     if ($changes) {
4373         PVE::QemuConfig->write_config($vmid, $conf);
4374     }
4375
4376     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4377
4378     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4379     foreach my $opt (sort keys %$pending_delete_hash) {
4380         next if $selection && !$selection->{$opt};
4381         my $force = $pending_delete_hash->{$opt}->{force};
4382         eval {
4383             if ($opt eq 'hotplug') {
4384                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4385             } elsif ($opt eq 'tablet') {
4386                 die "skip\n" if !$hotplug_features->{usb};
4387                 if ($defaults->{tablet}) {
4388                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4389                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4390                         if $arch eq 'aarch64';
4391                 } else {
4392                     vm_deviceunplug($vmid, $conf, 'tablet');
4393                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4394                 }
4395             } elsif ($opt =~ m/^usb\d+/) {
4396                 die "skip\n";
4397                 # since we cannot reliably hot unplug usb devices we are disabling it
4398                 #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4399                 #vm_deviceunplug($vmid, $conf, $opt);
4400             } elsif ($opt eq 'vcpus') {
4401                 die "skip\n" if !$hotplug_features->{cpu};
4402                 qemu_cpu_hotplug($vmid, $conf, undef);
4403             } elsif ($opt eq 'balloon') {
4404                 # enable balloon device is not hotpluggable
4405                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4406                 # here we reset the ballooning value to memory
4407                 my $balloon = $conf->{memory} || $defaults->{memory};
4408                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4409             } elsif ($fast_plug_option->{$opt}) {
4410                 # do nothing
4411             } elsif ($opt =~ m/^net(\d+)$/) {
4412                 die "skip\n" if !$hotplug_features->{network};
4413                 vm_deviceunplug($vmid, $conf, $opt);
4414             } elsif (is_valid_drivename($opt)) {
4415                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4416                 vm_deviceunplug($vmid, $conf, $opt);
4417                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4418             } elsif ($opt =~ m/^memory$/) {
4419                 die "skip\n" if !$hotplug_features->{memory};
4420                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4421             } elsif ($opt eq 'cpuunits') {
4422                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4423             } elsif ($opt eq 'cpulimit') {
4424                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4425             } else {
4426                 die "skip\n";
4427             }
4428         };
4429         if (my $err = $@) {
4430             &$add_error($opt, $err) if $err ne "skip\n";
4431         } else {
4432             delete $conf->{$opt};
4433             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4434         }
4435     }
4436
4437     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4438     $apply_pending_cloudinit = sub {
4439         return if $apply_pending_cloudinit_done; # once is enough
4440         $apply_pending_cloudinit_done = 1; # once is enough
4441
4442         my ($key, $value) = @_;
4443
4444         my @cloudinit_opts = keys %$confdesc_cloudinit;
4445         foreach my $opt (keys %{$conf->{pending}}) {
4446             next if !grep { $_ eq $opt } @cloudinit_opts;
4447             $conf->{$opt} = delete $conf->{pending}->{$opt};
4448         }
4449
4450         my $new_conf = { %$conf };
4451         $new_conf->{$key} = $value;
4452         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4453     };
4454
4455     foreach my $opt (keys %{$conf->{pending}}) {
4456         next if $selection && !$selection->{$opt};
4457         my $value = $conf->{pending}->{$opt};
4458         eval {
4459             if ($opt eq 'hotplug') {
4460                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4461             } elsif ($opt eq 'tablet') {
4462                 die "skip\n" if !$hotplug_features->{usb};
4463                 if ($value == 1) {
4464                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4465                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4466                         if $arch eq 'aarch64';
4467                 } elsif ($value == 0) {
4468                     vm_deviceunplug($vmid, $conf, 'tablet');
4469                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4470                 }
4471             } elsif ($opt =~ m/^usb\d+$/) {
4472                 die "skip\n";
4473                 # since we cannot reliably hot unplug usb devices we disable it for now
4474                 #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4475                 #my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4476                 #die "skip\n" if !$d;
4477                 #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4478             } elsif ($opt eq 'vcpus') {
4479                 die "skip\n" if !$hotplug_features->{cpu};
4480                 qemu_cpu_hotplug($vmid, $conf, $value);
4481             } elsif ($opt eq 'balloon') {
4482                 # enable/disable balloning device is not hotpluggable
4483                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4484                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4485                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4486
4487                 # allow manual ballooning if shares is set to zero
4488                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4489                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4490                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4491                 }
4492             } elsif ($opt =~ m/^net(\d+)$/) {
4493                 # some changes can be done without hotplug
4494                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4495                                     $vmid, $opt, $value, $arch, $machine_type);
4496             } elsif (is_valid_drivename($opt)) {
4497                 die "skip\n" if $opt eq 'efidisk0';
4498                 # some changes can be done without hotplug
4499                 my $drive = parse_drive($opt, $value);
4500                 if (drive_is_cloudinit($drive)) {
4501                     &$apply_pending_cloudinit($opt, $value);
4502                 }
4503                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4504                                      $vmid, $opt, $value, $arch, $machine_type);
4505             } elsif ($opt =~ m/^memory$/) { #dimms
4506                 die "skip\n" if !$hotplug_features->{memory};
4507                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4508             } elsif ($opt eq 'cpuunits') {
4509                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4510             } elsif ($opt eq 'cpulimit') {
4511                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4512                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4513             } else {
4514                 die "skip\n";  # skip non-hot-pluggable options
4515             }
4516         };
4517         if (my $err = $@) {
4518             &$add_error($opt, $err) if $err ne "skip\n";
4519         } else {
4520             $conf->{$opt} = $value;
4521             delete $conf->{pending}->{$opt};
4522         }
4523     }
4524
4525     PVE::QemuConfig->write_config($vmid, $conf);
4526 }
4527
4528 sub try_deallocate_drive {
4529     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4530
4531     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4532         my $volid = $drive->{file};
4533         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4534             my $sid = PVE::Storage::parse_volume_id($volid);
4535             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4536
4537             # check if the disk is really unused
4538             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4539                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4540             PVE::Storage::vdisk_free($storecfg, $volid);
4541             return 1;
4542         } else {
4543             # If vm is not owner of this disk remove from config
4544             return 1;
4545         }
4546     }
4547
4548     return;
4549 }
4550
4551 sub vmconfig_delete_or_detach_drive {
4552     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4553
4554     my $drive = parse_drive($opt, $conf->{$opt});
4555
4556     my $rpcenv = PVE::RPCEnvironment::get();
4557     my $authuser = $rpcenv->get_user();
4558
4559     if ($force) {
4560         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4561         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4562     } else {
4563         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4564     }
4565 }
4566
4567
4568
4569 sub vmconfig_apply_pending {
4570     my ($vmid, $conf, $storecfg, $errors) = @_;
4571
4572     my $add_apply_error = sub {
4573         my ($opt, $msg) = @_;
4574         my $err_msg = "unable to apply pending change $opt : $msg";
4575         $errors->{$opt} = $err_msg;
4576         warn $err_msg;
4577     };
4578
4579     # cold plug
4580
4581     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4582     foreach my $opt (sort keys %$pending_delete_hash) {
4583         my $force = $pending_delete_hash->{$opt}->{force};
4584         eval {
4585             if ($opt =~ m/^unused/) {
4586                 die "internal error";
4587             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4588                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4589             }
4590         };
4591         if (my $err = $@) {
4592             $add_apply_error->($opt, $err);
4593         } else {
4594             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4595             delete $conf->{$opt};
4596         }
4597     }
4598
4599     PVE::QemuConfig->cleanup_pending($conf);
4600
4601     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4602         next if $opt eq 'delete'; # just to be sure
4603         eval {
4604             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4605                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4606             }
4607         };
4608         if (my $err = $@) {
4609             $add_apply_error->($opt, $err);
4610         } else {
4611             $conf->{$opt} = delete $conf->{pending}->{$opt};
4612         }
4613     }
4614
4615     # write all changes at once to avoid unnecessary i/o
4616     PVE::QemuConfig->write_config($vmid, $conf);
4617 }
4618
4619 sub vmconfig_update_net {
4620     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4621
4622     my $newnet = parse_net($value);
4623
4624     if ($conf->{$opt}) {
4625         my $oldnet = parse_net($conf->{$opt});
4626
4627         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4628             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4629             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4630             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4631
4632             # for non online change, we try to hot-unplug
4633             die "skip\n" if !$hotplug;
4634             vm_deviceunplug($vmid, $conf, $opt);
4635         } else {
4636
4637             die "internal error" if $opt !~ m/net(\d+)/;
4638             my $iface = "tap${vmid}i$1";
4639
4640             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4641                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4642                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4643                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4644                 PVE::Network::tap_unplug($iface);
4645
4646                 if ($have_sdn) {
4647                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4648                 } else {
4649                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4650                 }
4651             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4652                 # Rate can be applied on its own but any change above needs to
4653                 # include the rate in tap_plug since OVS resets everything.
4654                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4655             }
4656
4657             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4658                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4659             }
4660
4661             return 1;
4662         }
4663     }
4664
4665     if ($hotplug) {
4666         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4667     } else {
4668         die "skip\n";
4669     }
4670 }
4671
4672 sub vmconfig_update_disk {
4673     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4674
4675     my $drive = parse_drive($opt, $value);
4676
4677     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
4678         my $media = $drive->{media} || 'disk';
4679         my $oldmedia = $old_drive->{media} || 'disk';
4680         die "unable to change media type\n" if $media ne $oldmedia;
4681
4682         if (!drive_is_cdrom($old_drive)) {
4683
4684             if ($drive->{file} ne $old_drive->{file}) {
4685
4686                 die "skip\n" if !$hotplug;
4687
4688                 # unplug and register as unused
4689                 vm_deviceunplug($vmid, $conf, $opt);
4690                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4691
4692             } else {
4693                 # update existing disk
4694
4695                 # skip non hotpluggable value
4696                 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4697                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4698                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4699                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
4700                     safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
4701                     die "skip\n";
4702                 }
4703
4704                 # apply throttle
4705                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4706                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4707                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4708                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4709                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4710                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4711                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4712                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4713                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4714                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4715                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4716                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4717                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4718                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4719                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4720                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4721                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4722                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4723
4724                     qemu_block_set_io_throttle(
4725                         $vmid,"drive-$opt",
4726                         ($drive->{mbps} || 0)*1024*1024,
4727                         ($drive->{mbps_rd} || 0)*1024*1024,
4728                         ($drive->{mbps_wr} || 0)*1024*1024,
4729                          $drive->{iops} || 0,
4730                          $drive->{iops_rd} || 0,
4731                          $drive->{iops_wr} || 0,
4732                         ($drive->{mbps_max} || 0)*1024*1024,
4733                         ($drive->{mbps_rd_max} || 0)*1024*1024,
4734                         ($drive->{mbps_wr_max} || 0)*1024*1024,
4735                          $drive->{iops_max} || 0,
4736                          $drive->{iops_rd_max} || 0,
4737                          $drive->{iops_wr_max} || 0,
4738                          $drive->{bps_max_length} || 1,
4739                          $drive->{bps_rd_max_length} || 1,
4740                          $drive->{bps_wr_max_length} || 1,
4741                          $drive->{iops_max_length} || 1,
4742                          $drive->{iops_rd_max_length} || 1,
4743                          $drive->{iops_wr_max_length} || 1,
4744                     );
4745
4746                 }
4747
4748                 return 1;
4749             }
4750
4751         } else { # cdrom
4752
4753             if ($drive->{file} eq 'none') {
4754                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4755                 if (drive_is_cloudinit($old_drive)) {
4756                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4757                 }
4758             } else {
4759                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4760
4761                 # force eject if locked
4762                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4763
4764                 if ($path) {
4765                     mon_cmd($vmid, "blockdev-change-medium",
4766                         id => "$opt", filename => "$path");
4767                 }
4768             }
4769
4770             return 1;
4771         }
4772     }
4773
4774     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4775     # hotplug new disks
4776     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4777     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
4778 }
4779
4780 # called in locked context by incoming migration
4781 sub vm_migrate_get_nbd_disks {
4782     my ($storecfg, $conf, $replicated_volumes) = @_;
4783
4784     my $local_volumes = {};
4785     PVE::QemuConfig->foreach_volume($conf, sub {
4786         my ($ds, $drive) = @_;
4787
4788         return if drive_is_cdrom($drive);
4789
4790         my $volid = $drive->{file};
4791
4792         return if !$volid;
4793
4794         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4795
4796         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4797         return if $scfg->{shared};
4798
4799         # replicated disks re-use existing state via bitmap
4800         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
4801         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
4802     });
4803     return $local_volumes;
4804 }
4805
4806 # called in locked context by incoming migration
4807 sub vm_migrate_alloc_nbd_disks {
4808     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
4809
4810     my $format = undef;
4811
4812     my $nbd = {};
4813     foreach my $opt (sort keys %$source_volumes) {
4814         my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
4815
4816         if ($use_existing) {
4817             $nbd->{$opt}->{drivestr} = print_drive($drive);
4818             $nbd->{$opt}->{volid} = $volid;
4819             $nbd->{$opt}->{replicated} = 1;
4820             next;
4821         }
4822
4823         # If a remote storage is specified and the format of the original
4824         # volume is not available there, fall back to the default format.
4825         # Otherwise use the same format as the original.
4826         if (!$storagemap->{identity}) {
4827             $storeid = map_storage($storagemap, $storeid);
4828             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4829             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4830             my $fileFormat = qemu_img_format($scfg, $volname);
4831             $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
4832         } else {
4833             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4834             $format = qemu_img_format($scfg, $volname);
4835         }
4836
4837         my $size = $drive->{size} / 1024;
4838         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
4839         my $newdrive = $drive;
4840         $newdrive->{format} = $format;
4841         $newdrive->{file} = $newvolid;
4842         my $drivestr = print_drive($newdrive);
4843         $nbd->{$opt}->{drivestr} = $drivestr;
4844         $nbd->{$opt}->{volid} = $newvolid;
4845     }
4846
4847     return $nbd;
4848 }
4849
4850 # see vm_start_nolock for parameters, additionally:
4851 # migrate_opts:
4852 #   storagemap = parsed storage map for allocating NBD disks
4853 sub vm_start {
4854     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
4855
4856     return PVE::QemuConfig->lock_config($vmid, sub {
4857         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
4858
4859         die "you can't start a vm if it's a template\n"
4860             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
4861
4862         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
4863         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
4864
4865         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
4866
4867         if ($has_backup_lock && $running) {
4868             # a backup is currently running, attempt to start the guest in the
4869             # existing QEMU instance
4870             return vm_resume($vmid);
4871         }
4872
4873         PVE::QemuConfig->check_lock($conf)
4874             if !($params->{skiplock} || $has_suspended_lock);
4875
4876         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
4877
4878         die "VM $vmid already running\n" if $running;
4879
4880         if (my $storagemap = $migrate_opts->{storagemap}) {
4881             my $replicated = $migrate_opts->{replicated_volumes};
4882             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
4883             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
4884
4885             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
4886                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
4887             }
4888         }
4889
4890         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
4891     });
4892 }
4893
4894
4895 # params:
4896 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
4897 #   skiplock => 0/1, skip checking for config lock
4898 #   skiptemplate => 0/1, skip checking whether VM is template
4899 #   forcemachine => to force Qemu machine (rollback/migration)
4900 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
4901 #   timeout => in seconds
4902 #   paused => start VM in paused state (backup)
4903 #   resume => resume from hibernation
4904 # migrate_opts:
4905 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
4906 #   migratedfrom => source node
4907 #   spice_ticket => used for spice migration, passed via tunnel/stdin
4908 #   network => CIDR of migration network
4909 #   type => secure/insecure - tunnel over encrypted connection or plain-text
4910 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
4911 #   replicated_volumes = which volids should be re-used with bitmaps for nbd migration
4912 sub vm_start_nolock {
4913     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
4914
4915     my $statefile = $params->{statefile};
4916     my $resume = $params->{resume};
4917
4918     my $migratedfrom = $migrate_opts->{migratedfrom};
4919     my $migration_type = $migrate_opts->{type};
4920
4921     my $res = {};
4922
4923     # clean up leftover reboot request files
4924     eval { clear_reboot_request($vmid); };
4925     warn $@ if $@;
4926
4927     if (!$statefile && scalar(keys %{$conf->{pending}})) {
4928         vmconfig_apply_pending($vmid, $conf, $storecfg);
4929         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4930     }
4931
4932     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4933
4934     my $defaults = load_defaults();
4935
4936     # set environment variable useful inside network script
4937     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4938
4939     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
4940
4941     my $forcemachine = $params->{forcemachine};
4942     my $forcecpu = $params->{forcecpu};
4943     if ($resume) {
4944         # enforce machine and CPU type on suspended vm to ensure HW compatibility
4945         $forcemachine = $conf->{runningmachine};
4946         $forcecpu = $conf->{runningcpu};
4947         print "Resuming suspended VM\n";
4948     }
4949
4950     my ($cmd, $vollist, $spice_port) =
4951         config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
4952
4953     my $migration_ip;
4954     my $get_migration_ip = sub {
4955         my ($nodename) = @_;
4956
4957         return $migration_ip if defined($migration_ip);
4958
4959         my $cidr = $migrate_opts->{network};
4960
4961         if (!defined($cidr)) {
4962             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4963             $cidr = $dc_conf->{migration}->{network};
4964         }
4965
4966         if (defined($cidr)) {
4967             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
4968
4969             die "could not get IP: no address configured on local " .
4970                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
4971
4972             die "could not get IP: multiple addresses configured on local " .
4973                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
4974
4975             $migration_ip = @$ips[0];
4976         }
4977
4978         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
4979             if !defined($migration_ip);
4980
4981         return $migration_ip;
4982     };
4983
4984     my $migrate_uri;
4985     if ($statefile) {
4986         if ($statefile eq 'tcp') {
4987             my $localip = "localhost";
4988             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4989             my $nodename = nodename();
4990
4991             if (!defined($migration_type)) {
4992                 if (defined($datacenterconf->{migration}->{type})) {
4993                     $migration_type = $datacenterconf->{migration}->{type};
4994                 } else {
4995                     $migration_type = 'secure';
4996                 }
4997             }
4998
4999             if ($migration_type eq 'insecure') {
5000                 $localip = $get_migration_ip->($nodename);
5001                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5002             }
5003
5004             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5005             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5006             $migrate_uri = "tcp:${localip}:${migrate_port}";
5007             push @$cmd, '-incoming', $migrate_uri;
5008             push @$cmd, '-S';
5009
5010         } elsif ($statefile eq 'unix') {
5011             # should be default for secure migrations as a ssh TCP forward
5012             # tunnel is not deterministic reliable ready and fails regurarly
5013             # to set up in time, so use UNIX socket forwards
5014             my $socket_addr = "/run/qemu-server/$vmid.migrate";
5015             unlink $socket_addr;
5016
5017             $migrate_uri = "unix:$socket_addr";
5018
5019             push @$cmd, '-incoming', $migrate_uri;
5020             push @$cmd, '-S';
5021
5022         } elsif (-e $statefile) {
5023             push @$cmd, '-loadstate', $statefile;
5024         } else {
5025             my $statepath = PVE::Storage::path($storecfg, $statefile);
5026             push @$vollist, $statefile;
5027             push @$cmd, '-loadstate', $statepath;
5028         }
5029     } elsif ($params->{paused}) {
5030         push @$cmd, '-S';
5031     }
5032
5033     # host pci devices
5034     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5035       my $d = parse_hostpci($conf->{"hostpci$i"});
5036       next if !$d;
5037       my $pcidevices = $d->{pciid};
5038       foreach my $pcidevice (@$pcidevices) {
5039             my $pciid = $pcidevice->{id};
5040
5041             my $info = PVE::SysFSTools::pci_device_info("$pciid");
5042             die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5043             die "no pci device info for device '$pciid'\n" if !$info;
5044
5045             if ($d->{mdev}) {
5046                 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5047                 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5048             } else {
5049                 die "can't unbind/bind PCI group to VFIO '$pciid'\n"
5050                     if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5051                 die "can't reset PCI device '$pciid'\n"
5052                     if $info->{has_fl_reset} && !PVE::SysFSTools::pci_dev_reset($info);
5053             }
5054       }
5055     }
5056
5057     PVE::Storage::activate_volumes($storecfg, $vollist);
5058
5059     eval {
5060         run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5061             outfunc => sub {}, errfunc => sub {});
5062     };
5063     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5064     # timeout should be more than enough here...
5065     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5066
5067     my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5068                                               : $defaults->{cpuunits};
5069
5070     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5071     my %run_params = (
5072         timeout => $statefile ? undef : $start_timeout,
5073         umask => 0077,
5074         noerr => 1,
5075     );
5076
5077     # when migrating, prefix QEMU output so other side can pick up any
5078     # errors that might occur and show the user
5079     if ($migratedfrom) {
5080         $run_params{quiet} = 1;
5081         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5082     }
5083
5084     my %properties = (
5085         Slice => 'qemu.slice',
5086         KillMode => 'none',
5087         CPUShares => $cpuunits
5088     );
5089
5090     if (my $cpulimit = $conf->{cpulimit}) {
5091         $properties{CPUQuota} = int($cpulimit * 100);
5092     }
5093     $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5094
5095     my $run_qemu = sub {
5096         PVE::Tools::run_fork sub {
5097             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5098
5099             my $exitcode = run_command($cmd, %run_params);
5100             die "QEMU exited with code $exitcode\n" if $exitcode;
5101         };
5102     };
5103
5104     if ($conf->{hugepages}) {
5105
5106         my $code = sub {
5107             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5108             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5109
5110             PVE::QemuServer::Memory::hugepages_mount();
5111             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5112
5113             eval { $run_qemu->() };
5114             if (my $err = $@) {
5115                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5116                     if !$conf->{keephugepages};
5117                 die $err;
5118             }
5119
5120             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5121                 if !$conf->{keephugepages};
5122         };
5123         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5124
5125     } else {
5126         eval { $run_qemu->() };
5127     }
5128
5129     if (my $err = $@) {
5130         # deactivate volumes if start fails
5131         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5132         die "start failed: $err";
5133     }
5134
5135     print "migration listens on $migrate_uri\n" if $migrate_uri;
5136     $res->{migrate_uri} = $migrate_uri;
5137
5138     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5139         eval { mon_cmd($vmid, "cont"); };
5140         warn $@ if $@;
5141     }
5142
5143     #start nbd server for storage migration
5144     if (my $nbd = $migrate_opts->{nbd}) {
5145         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5146
5147         my $migrate_storage_uri;
5148         # nbd_protocol_version > 0 for unix socket support
5149         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5150             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5151             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5152             $migrate_storage_uri = "nbd:unix:$socket_path";
5153         } else {
5154             my $nodename = nodename();
5155             my $localip = $get_migration_ip->($nodename);
5156             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5157             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5158
5159             mon_cmd($vmid, "nbd-server-start", addr => {
5160                 type => 'inet',
5161                 data => {
5162                     host => "${localip}",
5163                     port => "${storage_migrate_port}",
5164                 },
5165             });
5166             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5167             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5168         }
5169
5170         $res->{migrate_storage_uri} = $migrate_storage_uri;
5171
5172         foreach my $opt (sort keys %$nbd) {
5173             my $drivestr = $nbd->{$opt}->{drivestr};
5174             my $volid = $nbd->{$opt}->{volid};
5175             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5176             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5177             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5178             print "re-using replicated volume: $opt - $volid\n"
5179                 if $nbd->{$opt}->{replicated};
5180
5181             $res->{drives}->{$opt} = $nbd->{$opt};
5182             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5183         }
5184     }
5185
5186     if ($migratedfrom) {
5187         eval {
5188             set_migration_caps($vmid);
5189         };
5190         warn $@ if $@;
5191
5192         if ($spice_port) {
5193             print "spice listens on port $spice_port\n";
5194             $res->{spice_port} = $spice_port;
5195             if ($migrate_opts->{spice_ticket}) {
5196                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5197                     $migrate_opts->{spice_ticket});
5198                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5199             }
5200         }
5201
5202     } else {
5203         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5204             if !$statefile && $conf->{balloon};
5205
5206         foreach my $opt (keys %$conf) {
5207             next if $opt !~  m/^net\d+$/;
5208             my $nicconf = parse_net($conf->{$opt});
5209             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5210         }
5211     }
5212
5213     mon_cmd($vmid, 'qom-set',
5214                 path => "machine/peripheral/balloon0",
5215                 property => "guest-stats-polling-interval",
5216                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5217
5218     if ($resume) {
5219         print "Resumed VM, removing state\n";
5220         if (my $vmstate = $conf->{vmstate}) {
5221             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5222             PVE::Storage::vdisk_free($storecfg, $vmstate);
5223         }
5224         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5225         PVE::QemuConfig->write_config($vmid, $conf);
5226     }
5227
5228     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5229
5230     return $res;
5231 }
5232
5233 sub vm_commandline {
5234     my ($storecfg, $vmid, $snapname) = @_;
5235
5236     my $conf = PVE::QemuConfig->load_config($vmid);
5237     my $forcemachine;
5238     my $forcecpu;
5239
5240     if ($snapname) {
5241         my $snapshot = $conf->{snapshots}->{$snapname};
5242         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5243
5244         # check for machine or CPU overrides in snapshot
5245         $forcemachine = $snapshot->{runningmachine};
5246         $forcecpu = $snapshot->{runningcpu};
5247
5248         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5249
5250         $conf = $snapshot;
5251     }
5252
5253     my $defaults = load_defaults();
5254
5255     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults,
5256         $forcemachine, $forcecpu);
5257
5258     return PVE::Tools::cmd2string($cmd);
5259 }
5260
5261 sub vm_reset {
5262     my ($vmid, $skiplock) = @_;
5263
5264     PVE::QemuConfig->lock_config($vmid, sub {
5265
5266         my $conf = PVE::QemuConfig->load_config($vmid);
5267
5268         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5269
5270         mon_cmd($vmid, "system_reset");
5271     });
5272 }
5273
5274 sub get_vm_volumes {
5275     my ($conf) = @_;
5276
5277     my $vollist = [];
5278     foreach_volid($conf, sub {
5279         my ($volid, $attr) = @_;
5280
5281         return if $volid =~ m|^/|;
5282
5283         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5284         return if !$sid;
5285
5286         push @$vollist, $volid;
5287     });
5288
5289     return $vollist;
5290 }
5291
5292 sub vm_stop_cleanup {
5293     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5294
5295     eval {
5296
5297         if (!$keepActive) {
5298             my $vollist = get_vm_volumes($conf);
5299             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5300         }
5301
5302         foreach my $ext (qw(mon qmp pid vnc qga)) {
5303             unlink "/var/run/qemu-server/${vmid}.$ext";
5304         }
5305
5306         if ($conf->{ivshmem}) {
5307             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5308             # just delete it for now, VMs which have this already open do not
5309             # are affected, but new VMs will get a separated one. If this
5310             # becomes an issue we either add some sort of ref-counting or just
5311             # add a "don't delete on stop" flag to the ivshmem format.
5312             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5313         }
5314
5315         foreach my $key (keys %$conf) {
5316             next if $key !~ m/^hostpci(\d+)$/;
5317             my $hostpciindex = $1;
5318             my $d = parse_hostpci($conf->{$key});
5319             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5320
5321             foreach my $pci (@{$d->{pciid}}) {
5322                 my $pciid = $pci->{id};
5323                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5324             }
5325         }
5326
5327         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5328     };
5329     warn $@ if $@; # avoid errors - just warn
5330 }
5331
5332 # call only in locked context
5333 sub _do_vm_stop {
5334     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5335
5336     my $pid = check_running($vmid, $nocheck);
5337     return if !$pid;
5338
5339     my $conf;
5340     if (!$nocheck) {
5341         $conf = PVE::QemuConfig->load_config($vmid);
5342         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5343         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5344             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5345             $timeout = $opts->{down} if $opts->{down};
5346         }
5347         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5348     }
5349
5350     eval {
5351         if ($shutdown) {
5352             if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5353                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5354             } else {
5355                 mon_cmd($vmid, "system_powerdown");
5356             }
5357         } else {
5358             mon_cmd($vmid, "quit");
5359         }
5360     };
5361     my $err = $@;
5362
5363     if (!$err) {
5364         $timeout = 60 if !defined($timeout);
5365
5366         my $count = 0;
5367         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5368             $count++;
5369             sleep 1;
5370         }
5371
5372         if ($count >= $timeout) {
5373             if ($force) {
5374                 warn "VM still running - terminating now with SIGTERM\n";
5375                 kill 15, $pid;
5376             } else {
5377                 die "VM quit/powerdown failed - got timeout\n";
5378             }
5379         } else {
5380             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5381             return;
5382         }
5383     } else {
5384        if (!check_running($vmid, $nocheck)) {
5385             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5386             return;
5387        }
5388        if ($force) {
5389             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5390             kill 15, $pid;
5391         } else {
5392             die "VM quit/powerdown failed\n";
5393         }
5394     }
5395
5396     # wait again
5397     $timeout = 10;
5398
5399     my $count = 0;
5400     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5401         $count++;
5402         sleep 1;
5403     }
5404
5405     if ($count >= $timeout) {
5406         warn "VM still running - terminating now with SIGKILL\n";
5407         kill 9, $pid;
5408         sleep 1;
5409     }
5410
5411     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5412 }
5413
5414 # Note: use $nocheck to skip tests if VM configuration file exists.
5415 # We need that when migration VMs to other nodes (files already moved)
5416 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5417 sub vm_stop {
5418     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5419
5420     $force = 1 if !defined($force) && !$shutdown;
5421
5422     if ($migratedfrom){
5423         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5424         kill 15, $pid if $pid;
5425         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5426         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5427         return;
5428     }
5429
5430     PVE::QemuConfig->lock_config($vmid, sub {
5431         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5432    });
5433 }
5434
5435 sub vm_reboot {
5436     my ($vmid, $timeout) = @_;
5437
5438     PVE::QemuConfig->lock_config($vmid, sub {
5439         eval {
5440
5441             # only reboot if running, as qmeventd starts it again on a stop event
5442             return if !check_running($vmid);
5443
5444             create_reboot_request($vmid);
5445
5446             my $storecfg = PVE::Storage::config();
5447             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5448
5449         };
5450         if (my $err = $@) {
5451             # avoid that the next normal shutdown will be confused for a reboot
5452             clear_reboot_request($vmid);
5453             die $err;
5454         }
5455    });
5456 }
5457
5458 # note: if using the statestorage parameter, the caller has to check privileges
5459 sub vm_suspend {
5460     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5461
5462     my $conf;
5463     my $path;
5464     my $storecfg;
5465     my $vmstate;
5466
5467     PVE::QemuConfig->lock_config($vmid, sub {
5468
5469         $conf = PVE::QemuConfig->load_config($vmid);
5470
5471         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5472         PVE::QemuConfig->check_lock($conf)
5473             if !($skiplock || $is_backing_up);
5474
5475         die "cannot suspend to disk during backup\n"
5476             if $is_backing_up && $includestate;
5477
5478         if ($includestate) {
5479             $conf->{lock} = 'suspending';
5480             my $date = strftime("%Y-%m-%d", localtime(time()));
5481             $storecfg = PVE::Storage::config();
5482             if (!$statestorage) {
5483                 $statestorage = find_vmstate_storage($conf, $storecfg);
5484                 # check permissions for the storage
5485                 my $rpcenv = PVE::RPCEnvironment::get();
5486                 if ($rpcenv->{type} ne 'cli') {
5487                     my $authuser = $rpcenv->get_user();
5488                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5489                 }
5490             }
5491
5492
5493             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
5494                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5495             $path = PVE::Storage::path($storecfg, $vmstate);
5496             PVE::QemuConfig->write_config($vmid, $conf);
5497         } else {
5498             mon_cmd($vmid, "stop");
5499         }
5500     });
5501
5502     if ($includestate) {
5503         # save vm state
5504         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5505
5506         eval {
5507             mon_cmd($vmid, "savevm-start", statefile => $path);
5508             for(;;) {
5509                 my $state = mon_cmd($vmid, "query-savevm");
5510                 if (!$state->{status}) {
5511                     die "savevm not active\n";
5512                 } elsif ($state->{status} eq 'active') {
5513                     sleep(1);
5514                     next;
5515                 } elsif ($state->{status} eq 'completed') {
5516                     print "State saved, quitting\n";
5517                     last;
5518                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5519                     die "query-savevm failed with error '$state->{error}'\n"
5520                 } else {
5521                     die "query-savevm returned status '$state->{status}'\n";
5522                 }
5523             }
5524         };
5525         my $err = $@;
5526
5527         PVE::QemuConfig->lock_config($vmid, sub {
5528             $conf = PVE::QemuConfig->load_config($vmid);
5529             if ($err) {
5530                 # cleanup, but leave suspending lock, to indicate something went wrong
5531                 eval {
5532                     mon_cmd($vmid, "savevm-end");
5533                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5534                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5535                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
5536                     PVE::QemuConfig->write_config($vmid, $conf);
5537                 };
5538                 warn $@ if $@;
5539                 die $err;
5540             }
5541
5542             die "lock changed unexpectedly\n"
5543                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5544
5545             mon_cmd($vmid, "quit");
5546             $conf->{lock} = 'suspended';
5547             PVE::QemuConfig->write_config($vmid, $conf);
5548         });
5549     }
5550 }
5551
5552 sub vm_resume {
5553     my ($vmid, $skiplock, $nocheck) = @_;
5554
5555     PVE::QemuConfig->lock_config($vmid, sub {
5556         my $res = mon_cmd($vmid, 'query-status');
5557         my $resume_cmd = 'cont';
5558         my $reset = 0;
5559
5560         if ($res->{status}) {
5561             return if $res->{status} eq 'running'; # job done, go home
5562             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
5563             $reset = 1 if $res->{status} eq 'shutdown';
5564         }
5565
5566         if (!$nocheck) {
5567
5568             my $conf = PVE::QemuConfig->load_config($vmid);
5569
5570             PVE::QemuConfig->check_lock($conf)
5571                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5572         }
5573
5574         if ($reset) {
5575             # required if a VM shuts down during a backup and we get a resume
5576             # request before the backup finishes for example
5577             mon_cmd($vmid, "system_reset");
5578         }
5579         mon_cmd($vmid, $resume_cmd);
5580     });
5581 }
5582
5583 sub vm_sendkey {
5584     my ($vmid, $skiplock, $key) = @_;
5585
5586     PVE::QemuConfig->lock_config($vmid, sub {
5587
5588         my $conf = PVE::QemuConfig->load_config($vmid);
5589
5590         # there is no qmp command, so we use the human monitor command
5591         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5592         die $res if $res ne '';
5593     });
5594 }
5595
5596 # vzdump restore implementaion
5597
5598 sub tar_archive_read_firstfile {
5599     my $archive = shift;
5600
5601     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5602
5603     # try to detect archive type first
5604     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5605         die "unable to open file '$archive'\n";
5606     my $firstfile = <$fh>;
5607     kill 15, $pid;
5608     close $fh;
5609
5610     die "ERROR: archive contaions no data\n" if !$firstfile;
5611     chomp $firstfile;
5612
5613     return $firstfile;
5614 }
5615
5616 sub tar_restore_cleanup {
5617     my ($storecfg, $statfile) = @_;
5618
5619     print STDERR "starting cleanup\n";
5620
5621     if (my $fd = IO::File->new($statfile, "r")) {
5622         while (defined(my $line = <$fd>)) {
5623             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5624                 my $volid = $2;
5625                 eval {
5626                     if ($volid =~ m|^/|) {
5627                         unlink $volid || die 'unlink failed\n';
5628                     } else {
5629                         PVE::Storage::vdisk_free($storecfg, $volid);
5630                     }
5631                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5632                 };
5633                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5634             } else {
5635                 print STDERR "unable to parse line in statfile - $line";
5636             }
5637         }
5638         $fd->close();
5639     }
5640 }
5641
5642 sub restore_file_archive {
5643     my ($archive, $vmid, $user, $opts) = @_;
5644
5645     return restore_vma_archive($archive, $vmid, $user, $opts)
5646         if $archive eq '-';
5647
5648     my $info = PVE::Storage::archive_info($archive);
5649     my $format = $opts->{format} // $info->{format};
5650     my $comp = $info->{compression};
5651
5652     # try to detect archive format
5653     if ($format eq 'tar') {
5654         return restore_tar_archive($archive, $vmid, $user, $opts);
5655     } else {
5656         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5657     }
5658 }
5659
5660 # hepler to remove disks that will not be used after restore
5661 my $restore_cleanup_oldconf = sub {
5662     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
5663
5664     PVE::QemuConfig->foreach_volume($oldconf, sub {
5665         my ($ds, $drive) = @_;
5666
5667         return if drive_is_cdrom($drive, 1);
5668
5669         my $volid = $drive->{file};
5670         return if !$volid || $volid =~ m|^/|;
5671
5672         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
5673         return if !$path || !$owner || ($owner != $vmid);
5674
5675         # Note: only delete disk we want to restore
5676         # other volumes will become unused
5677         if ($virtdev_hash->{$ds}) {
5678             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
5679             if (my $err = $@) {
5680                 warn $err;
5681             }
5682         }
5683     });
5684
5685     # delete vmstate files, after the restore we have no snapshots anymore
5686     foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5687         my $snap = $oldconf->{snapshots}->{$snapname};
5688         if ($snap->{vmstate}) {
5689             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
5690             if (my $err = $@) {
5691                 warn $err;
5692             }
5693         }
5694     }
5695 };
5696
5697 # Helper to parse vzdump backup device hints
5698 #
5699 # $rpcenv: Environment, used to ckeck storage permissions
5700 # $user: User ID, to check storage permissions
5701 # $storecfg: Storage configuration
5702 # $fh: the file handle for reading the configuration
5703 # $devinfo: should contain device sizes for all backu-up'ed devices
5704 # $options: backup options (pool, default storage)
5705 #
5706 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
5707 my $parse_backup_hints = sub {
5708     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
5709
5710     my $virtdev_hash = {};
5711
5712     while (defined(my $line = <$fh>)) {
5713         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5714             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5715             die "archive does not contain data for drive '$virtdev'\n"
5716                 if !$devinfo->{$devname};
5717
5718             if (defined($options->{storage})) {
5719                 $storeid = $options->{storage} || 'local';
5720             } elsif (!$storeid) {
5721                 $storeid = 'local';
5722             }
5723             $format = 'raw' if !$format;
5724             $devinfo->{$devname}->{devname} = $devname;
5725             $devinfo->{$devname}->{virtdev} = $virtdev;
5726             $devinfo->{$devname}->{format} = $format;
5727             $devinfo->{$devname}->{storeid} = $storeid;
5728
5729             # check permission on storage
5730             my $pool = $options->{pool}; # todo: do we need that?
5731             if ($user ne 'root@pam') {
5732                 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5733             }
5734
5735             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5736         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
5737             my $virtdev = $1;
5738             my $drive = parse_drive($virtdev, $2);
5739             if (drive_is_cloudinit($drive)) {
5740                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5741                 $storeid = $options->{storage} if defined ($options->{storage});
5742                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5743                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
5744
5745                 $virtdev_hash->{$virtdev} = {
5746                     format => $format,
5747                     storeid => $storeid,
5748                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
5749                     is_cloudinit => 1,
5750                 };
5751             }
5752         }
5753     }
5754
5755     return $virtdev_hash;
5756 };
5757
5758 # Helper to allocate and activate all volumes required for a restore
5759 #
5760 # $storecfg: Storage configuration
5761 # $virtdev_hash: as returned by parse_backup_hints()
5762 #
5763 # Returns: { $virtdev => $volid }
5764 my $restore_allocate_devices = sub {
5765     my ($storecfg, $virtdev_hash, $vmid) = @_;
5766
5767     my $map = {};
5768     foreach my $virtdev (sort keys %$virtdev_hash) {
5769         my $d = $virtdev_hash->{$virtdev};
5770         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5771         my $storeid = $d->{storeid};
5772         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5773
5774         # test if requested format is supported
5775         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5776         my $supported = grep { $_ eq $d->{format} } @$validFormats;
5777         $d->{format} = $defFormat if !$supported;
5778
5779         my $name;
5780         if ($d->{is_cloudinit}) {
5781             $name = "vm-$vmid-cloudinit";
5782             $name .= ".$d->{format}" if $d->{format} ne 'raw';
5783         }
5784
5785         my $volid = PVE::Storage::vdisk_alloc(
5786             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
5787
5788         print STDERR "new volume ID is '$volid'\n";
5789         $d->{volid} = $volid;
5790
5791         PVE::Storage::activate_volumes($storecfg, [$volid]);
5792
5793         $map->{$virtdev} = $volid;
5794     }
5795
5796     return $map;
5797 };
5798
5799 my $restore_update_config_line = sub {
5800     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5801
5802     return if $line =~ m/^\#qmdump\#/;
5803     return if $line =~ m/^\#vzdump\#/;
5804     return if $line =~ m/^lock:/;
5805     return if $line =~ m/^unused\d+:/;
5806     return if $line =~ m/^parent:/;
5807
5808     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5809     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5810         # try to convert old 1.X settings
5811         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5812         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5813             my ($model, $macaddr) = split(/\=/, $devconfig);
5814             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5815             my $net = {
5816                 model => $model,
5817                 bridge => "vmbr$ind",
5818                 macaddr => $macaddr,
5819             };
5820             my $netstr = print_net($net);
5821
5822             print $outfd "net$cookie->{netcount}: $netstr\n";
5823             $cookie->{netcount}++;
5824         }
5825     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5826         my ($id, $netstr) = ($1, $2);
5827         my $net = parse_net($netstr);
5828         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5829         $netstr = print_net($net);
5830         print $outfd "$id: $netstr\n";
5831     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5832         my $virtdev = $1;
5833         my $value = $3;
5834         my $di = parse_drive($virtdev, $value);
5835         if (defined($di->{backup}) && !$di->{backup}) {
5836             print $outfd "#$line";
5837         } elsif ($map->{$virtdev}) {
5838             delete $di->{format}; # format can change on restore
5839             $di->{file} = $map->{$virtdev};
5840             $value = print_drive($di);
5841             print $outfd "$virtdev: $value\n";
5842         } else {
5843             print $outfd $line;
5844         }
5845     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5846         my $vmgenid = $1;
5847         if ($vmgenid ne '0') {
5848             # always generate a new vmgenid if there was a valid one setup
5849             $vmgenid = generate_uuid();
5850         }
5851         print $outfd "vmgenid: $vmgenid\n";
5852     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5853         my ($uuid, $uuid_str);
5854         UUID::generate($uuid);
5855         UUID::unparse($uuid, $uuid_str);
5856         my $smbios1 = parse_smbios1($2);
5857         $smbios1->{uuid} = $uuid_str;
5858         print $outfd $1.print_smbios1($smbios1)."\n";
5859     } else {
5860         print $outfd $line;
5861     }
5862 };
5863
5864 my $restore_deactivate_volumes = sub {
5865     my ($storecfg, $devinfo) = @_;
5866
5867     my $vollist = [];
5868     foreach my $devname (keys %$devinfo) {
5869         my $volid = $devinfo->{$devname}->{volid};
5870         push @$vollist, $volid if $volid;
5871     }
5872
5873     PVE::Storage::deactivate_volumes($storecfg, $vollist);
5874 };
5875
5876 my $restore_destroy_volumes = sub {
5877     my ($storecfg, $devinfo) = @_;
5878
5879     foreach my $devname (keys %$devinfo) {
5880         my $volid = $devinfo->{$devname}->{volid};
5881         next if !$volid;
5882         eval {
5883             if ($volid =~ m|^/|) {
5884                 unlink $volid || die 'unlink failed\n';
5885             } else {
5886                 PVE::Storage::vdisk_free($storecfg, $volid);
5887             }
5888             print STDERR "temporary volume '$volid' sucessfuly removed\n";
5889         };
5890         print STDERR "unable to cleanup '$volid' - $@" if $@;
5891     }
5892 };
5893
5894 sub scan_volids {
5895     my ($cfg, $vmid) = @_;
5896
5897     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5898
5899     my $volid_hash = {};
5900     foreach my $storeid (keys %$info) {
5901         foreach my $item (@{$info->{$storeid}}) {
5902             next if !($item->{volid} && $item->{size});
5903             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5904             $volid_hash->{$item->{volid}} = $item;
5905         }
5906     }
5907
5908     return $volid_hash;
5909 }
5910
5911 sub update_disk_config {
5912     my ($vmid, $conf, $volid_hash) = @_;
5913
5914     my $changes;
5915     my $prefix = "VM $vmid";
5916
5917     # used and unused disks
5918     my $referenced = {};
5919
5920     # Note: it is allowed to define multiple storages with same path (alias), so
5921     # we need to check both 'volid' and real 'path' (two different volid can point
5922     # to the same path).
5923
5924     my $referencedpath = {};
5925
5926     # update size info
5927     PVE::QemuConfig->foreach_volume($conf, sub {
5928         my ($opt, $drive) = @_;
5929
5930         my $volid = $drive->{file};
5931         return if !$volid;
5932         my $volume = $volid_hash->{$volid};
5933
5934         # mark volid as "in-use" for next step
5935         $referenced->{$volid} = 1;
5936         if ($volume && (my $path = $volume->{path})) {
5937             $referencedpath->{$path} = 1;
5938         }
5939
5940         return if drive_is_cdrom($drive);
5941         return if !$volume;
5942
5943         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
5944         if (defined($updated)) {
5945             $changes = 1;
5946             $conf->{$opt} = print_drive($updated);
5947             print "$prefix ($opt): $msg\n";
5948         }
5949     });
5950
5951     # remove 'unusedX' entry if volume is used
5952     PVE::QemuConfig->foreach_unused_volume($conf, sub {
5953         my ($opt, $drive) = @_;
5954
5955         my $volid = $drive->{file};
5956         return if !$volid;
5957
5958         my $path;
5959         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5960         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5961             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
5962             $changes = 1;
5963             delete $conf->{$opt};
5964         }
5965
5966         $referenced->{$volid} = 1;
5967         $referencedpath->{$path} = 1 if $path;
5968     });
5969
5970     foreach my $volid (sort keys %$volid_hash) {
5971         next if $volid =~ m/vm-$vmid-state-/;
5972         next if $referenced->{$volid};
5973         my $path = $volid_hash->{$volid}->{path};
5974         next if !$path; # just to be sure
5975         next if $referencedpath->{$path};
5976         $changes = 1;
5977         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
5978         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
5979         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5980     }
5981
5982     return $changes;
5983 }
5984
5985 sub rescan {
5986     my ($vmid, $nolock, $dryrun) = @_;
5987
5988     my $cfg = PVE::Storage::config();
5989
5990     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
5991     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
5992     foreach my $stor (keys %{$cfg->{ids}}) {
5993         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
5994     }
5995
5996     print "rescan volumes...\n";
5997     my $volid_hash = scan_volids($cfg, $vmid);
5998
5999     my $updatefn =  sub {
6000         my ($vmid) = @_;
6001
6002         my $conf = PVE::QemuConfig->load_config($vmid);
6003
6004         PVE::QemuConfig->check_lock($conf);
6005
6006         my $vm_volids = {};
6007         foreach my $volid (keys %$volid_hash) {
6008             my $info = $volid_hash->{$volid};
6009             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6010         }
6011
6012         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6013
6014         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6015     };
6016
6017     if (defined($vmid)) {
6018         if ($nolock) {
6019             &$updatefn($vmid);
6020         } else {
6021             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6022         }
6023     } else {
6024         my $vmlist = config_list();
6025         foreach my $vmid (keys %$vmlist) {
6026             if ($nolock) {
6027                 &$updatefn($vmid);
6028             } else {
6029                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6030             }
6031         }
6032     }
6033 }
6034
6035 sub restore_proxmox_backup_archive {
6036     my ($archive, $vmid, $user, $options) = @_;
6037
6038     my $storecfg = PVE::Storage::config();
6039
6040     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6041     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6042
6043     my $server = $scfg->{server};
6044     my $datastore = $scfg->{datastore};
6045     my $username = $scfg->{username} // 'root@pam';
6046     my $fingerprint = $scfg->{fingerprint};
6047     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6048
6049     my $repo = "$username\@$server:$datastore";
6050
6051     # This is only used for `pbs-restore`!
6052     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6053     local $ENV{PBS_PASSWORD} = $password;
6054     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6055
6056     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6057         PVE::Storage::parse_volname($storecfg, $archive);
6058
6059     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6060
6061     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6062
6063     my $tmpdir = "/var/tmp/vzdumptmp$$";
6064     rmtree $tmpdir;
6065     mkpath $tmpdir;
6066
6067     my $conffile = PVE::QemuConfig->config_file($vmid);
6068     my $tmpfn = "$conffile.$$.tmp";
6069      # disable interrupts (always do cleanups)
6070     local $SIG{INT} =
6071         local $SIG{TERM} =
6072         local $SIG{QUIT} =
6073         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6074
6075     # Note: $oldconf is undef if VM does not exists
6076     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6077     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6078
6079     my $rpcenv = PVE::RPCEnvironment::get();
6080     my $devinfo = {};
6081
6082     eval {
6083         # enable interrupts
6084         local $SIG{INT} =
6085             local $SIG{TERM} =
6086             local $SIG{QUIT} =
6087             local $SIG{HUP} =
6088             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6089
6090         my $cfgfn = "$tmpdir/qemu-server.conf";
6091         my $firewall_config_fn = "$tmpdir/fw.conf";
6092         my $index_fn = "$tmpdir/index.json";
6093
6094         my $cmd = "restore";
6095
6096         my $param = [$pbs_backup_name, "index.json", $index_fn];
6097         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6098         my $index = PVE::Tools::file_get_contents($index_fn);
6099         $index = decode_json($index);
6100
6101         # print Dumper($index);
6102         foreach my $info (@{$index->{files}}) {
6103             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6104                 my $devname = $1;
6105                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6106                     $devinfo->{$devname}->{size} = $1;
6107                 } else {
6108                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6109                 }
6110             }
6111         }
6112
6113         my $is_qemu_server_backup = scalar(
6114             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6115         );
6116         if (!$is_qemu_server_backup) {
6117             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6118         }
6119         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6120
6121         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6122         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6123
6124         if ($has_firewall_config) {
6125             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6126             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6127
6128             my $pve_firewall_dir = '/etc/pve/firewall';
6129             mkdir $pve_firewall_dir; # make sure the dir exists
6130             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6131         }
6132
6133         my $fh = IO::File->new($cfgfn, "r") ||
6134             die "unable to read qemu-server.conf - $!\n";
6135
6136         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6137
6138         # fixme: rate limit?
6139
6140         # create empty/temp config
6141         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6142
6143         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6144
6145         # allocate volumes
6146         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6147
6148         foreach my $virtdev (sort keys %$virtdev_hash) {
6149             my $d = $virtdev_hash->{$virtdev};
6150             next if $d->{is_cloudinit}; # no need to restore cloudinit
6151
6152             my $volid = $d->{volid};
6153
6154             my $path = PVE::Storage::path($storecfg, $volid);
6155
6156             # This is the ONLY user of the PBS_ env vars set on top of this function!
6157             my $pbs_restore_cmd = [
6158                 '/usr/bin/pbs-restore',
6159                 '--repository', $repo,
6160                 $pbs_backup_name,
6161                 "$d->{devname}.img.fidx",
6162                 $path,
6163                 '--verbose',
6164                 ];
6165
6166             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6167             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
6168
6169             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6170                 push @$pbs_restore_cmd, '--skip-zero';
6171             }
6172
6173             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6174             print "restore proxmox backup image: $dbg_cmdstring\n";
6175             run_command($pbs_restore_cmd);
6176         }
6177
6178         $fh->seek(0, 0) || die "seek failed - $!\n";
6179
6180         my $outfd = IO::File->new($tmpfn, "w") || die "unable to write config for VM $vmid\n";
6181
6182         my $cookie = { netcount => 0 };
6183         while (defined(my $line = <$fh>)) {
6184             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $options->{unique});
6185         }
6186
6187         $fh->close();
6188         $outfd->close();
6189     };
6190     my $err = $@;
6191
6192     $restore_deactivate_volumes->($storecfg, $devinfo);
6193
6194     rmtree $tmpdir;
6195
6196     if ($err) {
6197         unlink $tmpfn;
6198         $restore_destroy_volumes->($storecfg, $devinfo);
6199         die $err;
6200     }
6201
6202     rename($tmpfn, $conffile) ||
6203         die "unable to commit configuration file '$conffile'\n";
6204
6205     PVE::Cluster::cfs_update(); # make sure we read new file
6206
6207     eval { rescan($vmid, 1); };
6208     warn $@ if $@;
6209 }
6210
6211 sub restore_vma_archive {
6212     my ($archive, $vmid, $user, $opts, $comp) = @_;
6213
6214     my $readfrom = $archive;
6215
6216     my $cfg = PVE::Storage::config();
6217     my $commands = [];
6218     my $bwlimit = $opts->{bwlimit};
6219
6220     my $dbg_cmdstring = '';
6221     my $add_pipe = sub {
6222         my ($cmd) = @_;
6223         push @$commands, $cmd;
6224         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6225         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6226         $readfrom = '-';
6227     };
6228
6229     my $input = undef;
6230     if ($archive eq '-') {
6231         $input = '<&STDIN';
6232     } else {
6233         # If we use a backup from a PVE defined storage we also consider that
6234         # storage's rate limit:
6235         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6236         if (defined($volid)) {
6237             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6238             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6239             if ($readlimit) {
6240                 print STDERR "applying read rate limit: $readlimit\n";
6241                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6242                 $add_pipe->($cstream);
6243             }
6244         }
6245     }
6246
6247     if ($comp) {
6248         my $info = PVE::Storage::decompressor_info('vma', $comp);
6249         my $cmd = $info->{decompressor};
6250         push @$cmd, $readfrom;
6251         $add_pipe->($cmd);
6252     }
6253
6254     my $tmpdir = "/var/tmp/vzdumptmp$$";
6255     rmtree $tmpdir;
6256
6257     # disable interrupts (always do cleanups)
6258     local $SIG{INT} =
6259         local $SIG{TERM} =
6260         local $SIG{QUIT} =
6261         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6262
6263     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6264     POSIX::mkfifo($mapfifo, 0600);
6265     my $fifofh;
6266     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
6267
6268     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6269
6270     my $oldtimeout;
6271     my $timeout = 5;
6272
6273     my $devinfo = {};
6274
6275     my $rpcenv = PVE::RPCEnvironment::get();
6276
6277     my $conffile = PVE::QemuConfig->config_file($vmid);
6278     my $tmpfn = "$conffile.$$.tmp";
6279
6280     # Note: $oldconf is undef if VM does not exist
6281     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6282     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6283
6284     my %storage_limits;
6285
6286     my $print_devmap = sub {
6287         my $cfgfn = "$tmpdir/qemu-server.conf";
6288
6289         # we can read the config - that is already extracted
6290         my $fh = IO::File->new($cfgfn, "r") ||
6291             die "unable to read qemu-server.conf - $!\n";
6292
6293         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6294         if (-f $fwcfgfn) {
6295             my $pve_firewall_dir = '/etc/pve/firewall';
6296             mkdir $pve_firewall_dir; # make sure the dir exists
6297             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6298         }
6299
6300         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6301
6302         foreach my $key (keys %storage_limits) {
6303             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6304             next if !$limit;
6305             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6306             $storage_limits{$key} = $limit * 1024;
6307         }
6308
6309         foreach my $devname (keys %$devinfo) {
6310             die "found no device mapping information for device '$devname'\n"
6311                 if !$devinfo->{$devname}->{virtdev};
6312         }
6313
6314         # create empty/temp config
6315         if ($oldconf) {
6316             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6317             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6318         }
6319
6320         # allocate volumes
6321         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6322
6323         # print restore information to $fifofh
6324         foreach my $virtdev (sort keys %$virtdev_hash) {
6325             my $d = $virtdev_hash->{$virtdev};
6326             next if $d->{is_cloudinit}; # no need to restore cloudinit
6327
6328             my $storeid = $d->{storeid};
6329             my $volid = $d->{volid};
6330
6331             my $map_opts = '';
6332             if (my $limit = $storage_limits{$storeid}) {
6333                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6334             }
6335
6336             my $write_zeros = 1;
6337             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6338                 $write_zeros = 0;
6339             }
6340
6341             my $path = PVE::Storage::path($cfg, $volid);
6342
6343             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6344
6345             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6346         }
6347
6348         $fh->seek(0, 0) || die "seek failed - $!\n";
6349
6350         my $outfd = IO::File->new($tmpfn, "w") || die "unable to write config for VM $vmid\n";
6351
6352         my $cookie = { netcount => 0 };
6353         while (defined(my $line = <$fh>)) {
6354             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6355         }
6356
6357         $fh->close();
6358         $outfd->close();
6359     };
6360
6361     eval {
6362         # enable interrupts
6363         local $SIG{INT} =
6364             local $SIG{TERM} =
6365             local $SIG{QUIT} =
6366             local $SIG{HUP} =
6367             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6368         local $SIG{ALRM} = sub { die "got timeout\n"; };
6369
6370         $oldtimeout = alarm($timeout);
6371
6372         my $parser = sub {
6373             my $line = shift;
6374
6375             print "$line\n";
6376
6377             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6378                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6379                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6380             } elsif ($line =~ m/^CTIME: /) {
6381                 # we correctly received the vma config, so we can disable
6382                 # the timeout now for disk allocation (set to 10 minutes, so
6383                 # that we always timeout if something goes wrong)
6384                 alarm(600);
6385                 &$print_devmap();
6386                 print $fifofh "done\n";
6387                 my $tmp = $oldtimeout || 0;
6388                 $oldtimeout = undef;
6389                 alarm($tmp);
6390                 close($fifofh);
6391                 $fifofh = undef;
6392             }
6393         };
6394
6395         print "restore vma archive: $dbg_cmdstring\n";
6396         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6397     };
6398     my $err = $@;
6399
6400     alarm($oldtimeout) if $oldtimeout;
6401
6402     $restore_deactivate_volumes->($cfg, $devinfo);
6403
6404     close($fifofh) if $fifofh;
6405     unlink $mapfifo;
6406     rmtree $tmpdir;
6407
6408     if ($err) {
6409         unlink $tmpfn;
6410         $restore_destroy_volumes->($cfg, $devinfo);
6411         die $err;
6412     }
6413
6414     rename($tmpfn, $conffile) ||
6415         die "unable to commit configuration file '$conffile'\n";
6416
6417     PVE::Cluster::cfs_update(); # make sure we read new file
6418
6419     eval { rescan($vmid, 1); };
6420     warn $@ if $@;
6421 }
6422
6423 sub restore_tar_archive {
6424     my ($archive, $vmid, $user, $opts) = @_;
6425
6426     if ($archive ne '-') {
6427         my $firstfile = tar_archive_read_firstfile($archive);
6428         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
6429             if $firstfile ne 'qemu-server.conf';
6430     }
6431
6432     my $storecfg = PVE::Storage::config();
6433
6434     # avoid zombie disks when restoring over an existing VM -> cleanup first
6435     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6436     # skiplock=1 because qmrestore has set the 'create' lock itself already
6437     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6438     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6439
6440     my $tocmd = "/usr/lib/qemu-server/qmextract";
6441
6442     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6443     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6444     $tocmd .= ' --prealloc' if $opts->{prealloc};
6445     $tocmd .= ' --info' if $opts->{info};
6446
6447     # tar option "xf" does not autodetect compression when read from STDIN,
6448     # so we pipe to zcat
6449     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6450         PVE::Tools::shellquote("--to-command=$tocmd");
6451
6452     my $tmpdir = "/var/tmp/vzdumptmp$$";
6453     mkpath $tmpdir;
6454
6455     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6456     local $ENV{VZDUMP_VMID} = $vmid;
6457     local $ENV{VZDUMP_USER} = $user;
6458
6459     my $conffile = PVE::QemuConfig->config_file($vmid);
6460     my $tmpfn = "$conffile.$$.tmp";
6461
6462     # disable interrupts (always do cleanups)
6463     local $SIG{INT} =
6464         local $SIG{TERM} =
6465         local $SIG{QUIT} =
6466         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6467
6468     eval {
6469         # enable interrupts
6470         local $SIG{INT} =
6471             local $SIG{TERM} =
6472             local $SIG{QUIT} =
6473             local $SIG{HUP} =
6474             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6475
6476         if ($archive eq '-') {
6477             print "extracting archive from STDIN\n";
6478             run_command($cmd, input => "<&STDIN");
6479         } else {
6480             print "extracting archive '$archive'\n";
6481             run_command($cmd);
6482         }
6483
6484         return if $opts->{info};
6485
6486         # read new mapping
6487         my $map = {};
6488         my $statfile = "$tmpdir/qmrestore.stat";
6489         if (my $fd = IO::File->new($statfile, "r")) {
6490             while (defined (my $line = <$fd>)) {
6491                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6492                     $map->{$1} = $2 if $1;
6493                 } else {
6494                     print STDERR "unable to parse line in statfile - $line\n";
6495                 }
6496             }
6497             $fd->close();
6498         }
6499
6500         my $confsrc = "$tmpdir/qemu-server.conf";
6501
6502         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
6503
6504         my $outfd = IO::File->new($tmpfn, "w") || die "unable to write config for VM $vmid\n";
6505
6506         my $cookie = { netcount => 0 };
6507         while (defined (my $line = <$srcfd>)) {
6508             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6509         }
6510
6511         $srcfd->close();
6512         $outfd->close();
6513     };
6514     if (my $err = $@) {
6515         unlink $tmpfn;
6516         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6517         die $err;
6518     }
6519
6520     rmtree $tmpdir;
6521
6522     rename $tmpfn, $conffile ||
6523         die "unable to commit configuration file '$conffile'\n";
6524
6525     PVE::Cluster::cfs_update(); # make sure we read new file
6526
6527     eval { rescan($vmid, 1); };
6528     warn $@ if $@;
6529 };
6530
6531 sub foreach_storage_used_by_vm {
6532     my ($conf, $func) = @_;
6533
6534     my $sidhash = {};
6535
6536     PVE::QemuConfig->foreach_volume($conf, sub {
6537         my ($ds, $drive) = @_;
6538         return if drive_is_cdrom($drive);
6539
6540         my $volid = $drive->{file};
6541
6542         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6543         $sidhash->{$sid} = $sid if $sid;
6544     });
6545
6546     foreach my $sid (sort keys %$sidhash) {
6547         &$func($sid);
6548     }
6549 }
6550
6551 my $qemu_snap_storage = {
6552     rbd => 1,
6553 };
6554 sub do_snapshots_with_qemu {
6555     my ($storecfg, $volid) = @_;
6556
6557     my $storage_name = PVE::Storage::parse_volume_id($volid);
6558     my $scfg = $storecfg->{ids}->{$storage_name};
6559     die "could not find storage '$storage_name'\n" if !defined($scfg);
6560
6561     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6562         return 1;
6563     }
6564
6565     if ($volid =~ m/\.(qcow2|qed)$/){
6566         return 1;
6567     }
6568
6569     return;
6570 }
6571
6572 sub qga_check_running {
6573     my ($vmid, $nowarn) = @_;
6574
6575     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6576     if ($@) {
6577         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6578         return 0;
6579     }
6580     return 1;
6581 }
6582
6583 sub template_create {
6584     my ($vmid, $conf, $disk) = @_;
6585
6586     my $storecfg = PVE::Storage::config();
6587
6588     PVE::QemuConfig->foreach_volume($conf, sub {
6589         my ($ds, $drive) = @_;
6590
6591         return if drive_is_cdrom($drive);
6592         return if $disk && $ds ne $disk;
6593
6594         my $volid = $drive->{file};
6595         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6596
6597         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6598         $drive->{file} = $voliddst;
6599         $conf->{$ds} = print_drive($drive);
6600         PVE::QemuConfig->write_config($vmid, $conf);
6601     });
6602 }
6603
6604 sub convert_iscsi_path {
6605     my ($path) = @_;
6606
6607     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6608         my $portal = $1;
6609         my $target = $2;
6610         my $lun = $3;
6611
6612         my $initiator_name = get_initiator_name();
6613
6614         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6615                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6616     }
6617
6618     die "cannot convert iscsi path '$path', unkown format\n";
6619 }
6620
6621 sub qemu_img_convert {
6622     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6623
6624     my $storecfg = PVE::Storage::config();
6625     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6626     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6627
6628     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6629
6630     my $cachemode;
6631     my $src_path;
6632     my $src_is_iscsi = 0;
6633     my $src_format;
6634
6635     if ($src_storeid) {
6636         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6637         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6638         $src_format = qemu_img_format($src_scfg, $src_volname);
6639         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6640         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6641         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6642     } elsif (-f $src_volid) {
6643         $src_path = $src_volid;
6644         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6645             $src_format = $1;
6646         }
6647     }
6648
6649     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6650
6651     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6652     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6653     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6654     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6655
6656     my $cmd = [];
6657     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6658     push @$cmd, '-l', "snapshot.name=$snapname"
6659         if $snapname && $src_format && $src_format eq "qcow2";
6660     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6661     push @$cmd, '-T', $cachemode if defined($cachemode);
6662
6663     if ($src_is_iscsi) {
6664         push @$cmd, '--image-opts';
6665         $src_path = convert_iscsi_path($src_path);
6666     } elsif ($src_format) {
6667         push @$cmd, '-f', $src_format;
6668     }
6669
6670     if ($dst_is_iscsi) {
6671         push @$cmd, '--target-image-opts';
6672         $dst_path = convert_iscsi_path($dst_path);
6673     } else {
6674         push @$cmd, '-O', $dst_format;
6675     }
6676
6677     push @$cmd, $src_path;
6678
6679     if (!$dst_is_iscsi && $is_zero_initialized) {
6680         push @$cmd, "zeroinit:$dst_path";
6681     } else {
6682         push @$cmd, $dst_path;
6683     }
6684
6685     my $parser = sub {
6686         my $line = shift;
6687         if($line =~ m/\((\S+)\/100\%\)/){
6688             my $percent = $1;
6689             my $transferred = int($size * $percent / 100);
6690             my $remaining = $size - $transferred;
6691
6692             print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6693         }
6694
6695     };
6696
6697     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6698     my $err = $@;
6699     die "copy failed: $err" if $err;
6700 }
6701
6702 sub qemu_img_format {
6703     my ($scfg, $volname) = @_;
6704
6705     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6706         return $1;
6707     } else {
6708         return "raw";
6709     }
6710 }
6711
6712 sub qemu_drive_mirror {
6713     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
6714
6715     $jobs = {} if !$jobs;
6716
6717     my $qemu_target;
6718     my $format;
6719     $jobs->{"drive-$drive"} = {};
6720
6721     if ($dst_volid =~ /^nbd:/) {
6722         $qemu_target = $dst_volid;
6723         $format = "nbd";
6724     } else {
6725         my $storecfg = PVE::Storage::config();
6726         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6727
6728         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6729
6730         $format = qemu_img_format($dst_scfg, $dst_volname);
6731
6732         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6733
6734         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6735     }
6736
6737     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6738     $opts->{format} = $format if $format;
6739
6740     if (defined($src_bitmap)) {
6741         $opts->{sync} = 'incremental';
6742         $opts->{bitmap} = $src_bitmap;
6743         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
6744     }
6745
6746     if (defined($bwlimit)) {
6747         $opts->{speed} = $bwlimit * 1024;
6748         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6749     } else {
6750         print "drive mirror is starting for drive-$drive\n";
6751     }
6752
6753     # if a job already runs for this device we get an error, catch it for cleanup
6754     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
6755     if (my $err = $@) {
6756         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6757         warn "$@\n" if $@;
6758         die "mirroring error: $err\n";
6759     }
6760
6761     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
6762 }
6763
6764 # $completion can be either
6765 # 'complete': wait until all jobs are ready, block-job-complete them (default)
6766 # 'cancel': wait until all jobs are ready, block-job-cancel them
6767 # 'skip': wait until all jobs are ready, return with block jobs in ready state
6768 sub qemu_drive_mirror_monitor {
6769     my ($vmid, $vmiddst, $jobs, $completion, $qga) = @_;
6770
6771     $completion //= 'complete';
6772
6773     eval {
6774         my $err_complete = 0;
6775
6776         while (1) {
6777             die "storage migration timed out\n" if $err_complete > 300;
6778
6779             my $stats = mon_cmd($vmid, "query-block-jobs");
6780
6781             my $running_mirror_jobs = {};
6782             foreach my $stat (@$stats) {
6783                 next if $stat->{type} ne 'mirror';
6784                 $running_mirror_jobs->{$stat->{device}} = $stat;
6785             }
6786
6787             my $readycounter = 0;
6788
6789             foreach my $job (keys %$jobs) {
6790
6791                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6792                     print "$job : finished\n";
6793                     delete $jobs->{$job};
6794                     next;
6795                 }
6796
6797                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6798
6799                 my $busy = $running_mirror_jobs->{$job}->{busy};
6800                 my $ready = $running_mirror_jobs->{$job}->{ready};
6801                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6802                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6803                     my $remaining = $total - $transferred;
6804                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6805
6806                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6807                 }
6808
6809                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6810             }
6811
6812             last if scalar(keys %$jobs) == 0;
6813
6814             if ($readycounter == scalar(keys %$jobs)) {
6815                 print "all mirroring jobs are ready \n";
6816                 last if $completion eq 'skip'; #do the complete later
6817
6818                 if ($vmiddst && $vmiddst != $vmid) {
6819                     my $agent_running = $qga && qga_check_running($vmid);
6820                     if ($agent_running) {
6821                         print "freeze filesystem\n";
6822                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6823                     } else {
6824                         print "suspend vm\n";
6825                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6826                     }
6827
6828                     # if we clone a disk for a new target vm, we don't switch the disk
6829                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6830
6831                     if ($agent_running) {
6832                         print "unfreeze filesystem\n";
6833                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6834                     } else {
6835                         print "resume vm\n";
6836                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6837                     }
6838
6839                     last;
6840                 } else {
6841
6842                     foreach my $job (keys %$jobs) {
6843                         # try to switch the disk if source and destination are on the same guest
6844                         print "$job: Completing block job...\n";
6845
6846                         my $op;
6847                         if ($completion eq 'complete') {
6848                             $op = 'block-job-complete';
6849                         } elsif ($completion eq 'cancel') {
6850                             $op = 'block-job-cancel';
6851                         } else {
6852                             die "invalid completion value: $completion\n";
6853                         }
6854                         eval { mon_cmd($vmid, $op, device => $job) };
6855                         if ($@ =~ m/cannot be completed/) {
6856                             print "$job: Block job cannot be completed, try again.\n";
6857                             $err_complete++;
6858                         }else {
6859                             print "$job: Completed successfully.\n";
6860                             $jobs->{$job}->{complete} = 1;
6861                         }
6862                     }
6863                 }
6864             }
6865             sleep 1;
6866         }
6867     };
6868     my $err = $@;
6869
6870     if ($err) {
6871         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6872         die "mirroring error: $err";
6873     }
6874
6875 }
6876
6877 sub qemu_blockjobs_cancel {
6878     my ($vmid, $jobs) = @_;
6879
6880     foreach my $job (keys %$jobs) {
6881         print "$job: Cancelling block job\n";
6882         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
6883         $jobs->{$job}->{cancel} = 1;
6884     }
6885
6886     while (1) {
6887         my $stats = mon_cmd($vmid, "query-block-jobs");
6888
6889         my $running_jobs = {};
6890         foreach my $stat (@$stats) {
6891             $running_jobs->{$stat->{device}} = $stat;
6892         }
6893
6894         foreach my $job (keys %$jobs) {
6895
6896             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6897                 print "$job: Done.\n";
6898                 delete $jobs->{$job};
6899             }
6900         }
6901
6902         last if scalar(keys %$jobs) == 0;
6903
6904         sleep 1;
6905     }
6906 }
6907
6908 sub clone_disk {
6909     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6910         $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
6911
6912     my $newvolid;
6913
6914     if (!$full) {
6915         print "create linked clone of drive $drivename ($drive->{file})\n";
6916         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6917         push @$newvollist, $newvolid;
6918     } else {
6919
6920         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6921         $storeid = $storage if $storage;
6922
6923         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6924
6925         print "create full clone of drive $drivename ($drive->{file})\n";
6926         my $name = undef;
6927         my $size = undef;
6928         if (drive_is_cloudinit($drive)) {
6929             $name = "vm-$newvmid-cloudinit";
6930             $name .= ".$dst_format" if $dst_format ne 'raw';
6931             $snapname = undef;
6932             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
6933         } elsif ($drivename eq 'efidisk0') {
6934             $size = get_efivars_size($conf);
6935         } else {
6936             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6937         }
6938         $size /= 1024;
6939         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, $size);
6940         push @$newvollist, $newvolid;
6941
6942         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6943
6944         if (drive_is_cloudinit($drive)) {
6945             goto no_data_clone;
6946         }
6947
6948         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6949         if (!$running || $snapname) {
6950             # TODO: handle bwlimits
6951             if ($drivename eq 'efidisk0') {
6952                 # the relevant data on the efidisk may be smaller than the source
6953                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
6954                 # that is given by the OVMF_VARS.fd
6955                 my $src_path = PVE::Storage::path($storecfg, $drive->{file});
6956                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
6957                 run_command(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=1", "count=$size",
6958                     "if=$src_path", "of=$dst_path"]);
6959             } else {
6960                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6961             }
6962         } else {
6963
6964             my $kvmver = get_running_qemu_version ($vmid);
6965             if (!min_version($kvmver, 2, 7)) {
6966                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6967                     if $drive->{iothread};
6968             }
6969
6970             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs,
6971                 $completion, $qga, $bwlimit);
6972         }
6973     }
6974
6975 no_data_clone:
6976     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6977
6978     my $disk = $drive;
6979     $disk->{format} = undef;
6980     $disk->{file} = $newvolid;
6981     $disk->{size} = $size;
6982
6983     return $disk;
6984 }
6985
6986 sub get_running_qemu_version {
6987     my ($vmid) = @_;
6988     my $res = mon_cmd($vmid, "query-version");
6989     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6990 }
6991
6992 sub qemu_use_old_bios_files {
6993     my ($machine_type) = @_;
6994
6995     return if !$machine_type;
6996
6997     my $use_old_bios_files = undef;
6998
6999     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7000         $machine_type = $1;
7001         $use_old_bios_files = 1;
7002     } else {
7003         my $version = extract_version($machine_type, kvm_user_version());
7004         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7005         # load new efi bios files on migration. So this hack is required to allow
7006         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7007         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7008         $use_old_bios_files = !min_version($version, 2, 4);
7009     }
7010
7011     return ($use_old_bios_files, $machine_type);
7012 }
7013
7014 sub get_efivars_size {
7015     my ($conf) = @_;
7016     my $arch = get_vm_arch($conf);
7017     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7018     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
7019     return -s $ovmf_vars;
7020 }
7021
7022 sub update_efidisk_size {
7023     my ($conf) = @_;
7024
7025     return if !defined($conf->{efidisk0});
7026
7027     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
7028     $disk->{size} = get_efivars_size($conf);
7029     $conf->{efidisk0} = print_drive($disk);
7030
7031     return;
7032 }
7033
7034 sub create_efidisk($$$$$) {
7035     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7036
7037     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7038     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7039
7040     my $vars_size_b = -s $ovmf_vars;
7041     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7042     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7043     PVE::Storage::activate_volumes($storecfg, [$volid]);
7044
7045     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7046     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7047
7048     return ($volid, $size/1024);
7049 }
7050
7051 sub vm_iothreads_list {
7052     my ($vmid) = @_;
7053
7054     my $res = mon_cmd($vmid, 'query-iothreads');
7055
7056     my $iothreads = {};
7057     foreach my $iothread (@$res) {
7058         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7059     }
7060
7061     return $iothreads;
7062 }
7063
7064 sub scsihw_infos {
7065     my ($conf, $drive) = @_;
7066
7067     my $maxdev = 0;
7068
7069     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7070         $maxdev = 7;
7071     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7072         $maxdev = 1;
7073     } else {
7074         $maxdev = 256;
7075     }
7076
7077     my $controller = int($drive->{index} / $maxdev);
7078     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
7079         ? "virtioscsi"
7080         : "scsihw";
7081
7082     return ($maxdev, $controller, $controller_prefix);
7083 }
7084
7085 sub windows_version {
7086     my ($ostype) = @_;
7087
7088     return 0 if !$ostype;
7089
7090     my $winversion = 0;
7091
7092     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7093         $winversion = 5;
7094     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7095         $winversion = 6;
7096     } elsif ($ostype =~ m/^win(\d+)$/) {
7097         $winversion = $1;
7098     }
7099
7100     return $winversion;
7101 }
7102
7103 sub resolve_dst_disk_format {
7104         my ($storecfg, $storeid, $src_volname, $format) = @_;
7105         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7106
7107         if (!$format) {
7108             # if no target format is specified, use the source disk format as hint
7109             if ($src_volname) {
7110                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7111                 $format = qemu_img_format($scfg, $src_volname);
7112             } else {
7113                 return $defFormat;
7114             }
7115         }
7116
7117         # test if requested format is supported - else use default
7118         my $supported = grep { $_ eq $format } @$validFormats;
7119         $format = $defFormat if !$supported;
7120         return $format;
7121 }
7122
7123 # NOTE: if this logic changes, please update docs & possibly gui logic
7124 sub find_vmstate_storage {
7125     my ($conf, $storecfg) = @_;
7126
7127     # first, return storage from conf if set
7128     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7129
7130     my ($target, $shared, $local);
7131
7132     foreach_storage_used_by_vm($conf, sub {
7133         my ($sid) = @_;
7134         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7135         my $dst = $scfg->{shared} ? \$shared : \$local;
7136         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7137     });
7138
7139     # second, use shared storage where VM has at least one disk
7140     # third, use local storage where VM has at least one disk
7141     # fall back to local storage
7142     $target = $shared // $local // 'local';
7143
7144     return $target;
7145 }
7146
7147 sub generate_uuid {
7148     my ($uuid, $uuid_str);
7149     UUID::generate($uuid);
7150     UUID::unparse($uuid, $uuid_str);
7151     return $uuid_str;
7152 }
7153
7154 sub generate_smbios1_uuid {
7155     return "uuid=".generate_uuid();
7156 }
7157
7158 sub nbd_stop {
7159     my ($vmid) = @_;
7160
7161     mon_cmd($vmid, 'nbd-server-stop');
7162 }
7163
7164 sub create_reboot_request {
7165     my ($vmid) = @_;
7166     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7167         or die "failed to create reboot trigger file: $!\n";
7168     close($fh);
7169 }
7170
7171 sub clear_reboot_request {
7172     my ($vmid) = @_;
7173     my $path = "/run/qemu-server/$vmid.reboot";
7174     my $res = 0;
7175
7176     $res = unlink($path);
7177     die "could not remove reboot request for $vmid: $!"
7178         if !$res && $! != POSIX::ENOENT;
7179
7180     return $res;
7181 }
7182
7183 sub bootorder_from_legacy {
7184     my ($conf, $bootcfg) = @_;
7185
7186     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
7187     my $bootindex_hash = {};
7188     my $i = 1;
7189     foreach my $o (split(//, $boot)) {
7190         $bootindex_hash->{$o} = $i*100;
7191         $i++;
7192     }
7193
7194     my $bootorder = {};
7195
7196     PVE::QemuConfig->foreach_volume($conf, sub {
7197         my ($ds, $drive) = @_;
7198
7199         if (drive_is_cdrom ($drive, 1)) {
7200             if ($bootindex_hash->{d}) {
7201                 $bootorder->{$ds} = $bootindex_hash->{d};
7202                 $bootindex_hash->{d} += 1;
7203             }
7204         } elsif ($bootindex_hash->{c}) {
7205             $bootorder->{$ds} = $bootindex_hash->{c}
7206                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
7207             $bootindex_hash->{c} += 1;
7208         }
7209     });
7210
7211     if ($bootindex_hash->{n}) {
7212         for (my $i = 0; $i < $MAX_NETS; $i++) {
7213             my $netname = "net$i";
7214             next if !$conf->{$netname};
7215             $bootorder->{$netname} = $bootindex_hash->{n};
7216             $bootindex_hash->{n} += 1;
7217         }
7218     }
7219
7220     return $bootorder;
7221 }
7222
7223 # Generate default device list for 'boot: order=' property. Matches legacy
7224 # default boot order, but with explicit device names. This is important, since
7225 # the fallback for when neither 'order' nor the old format is specified relies
7226 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
7227 sub get_default_bootdevices {
7228     my ($conf) = @_;
7229
7230     my @ret = ();
7231
7232     # harddisk
7233     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
7234     push @ret, $first if $first;
7235
7236     # cdrom
7237     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
7238     push @ret, $first if $first;
7239
7240     # network
7241     for (my $i = 0; $i < $MAX_NETS; $i++) {
7242         my $netname = "net$i";
7243         next if !$conf->{$netname};
7244         push @ret, $netname;
7245         last;
7246     }
7247
7248     return \@ret;
7249 }
7250
7251 sub device_bootorder {
7252     my ($conf) = @_;
7253
7254     return bootorder_from_legacy($conf) if !defined($conf->{boot});
7255
7256     my $boot = parse_property_string($boot_fmt, $conf->{boot});
7257
7258     my $bootorder = {};
7259     if (!defined($boot) || $boot->{legacy}) {
7260         $bootorder = bootorder_from_legacy($conf, $boot);
7261     } elsif ($boot->{order}) {
7262         my $i = 100; # start at 100 to allow user to insert devices before us with -args
7263         for my $dev (PVE::Tools::split_list($boot->{order})) {
7264             $bootorder->{$dev} = $i++;
7265         }
7266     }
7267
7268     return $bootorder;
7269 }
7270
7271 # bash completion helper
7272
7273 sub complete_backup_archives {
7274     my ($cmdname, $pname, $cvalue) = @_;
7275
7276     my $cfg = PVE::Storage::config();
7277
7278     my $storeid;
7279
7280     if ($cvalue =~ m/^([^:]+):/) {
7281         $storeid = $1;
7282     }
7283
7284     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7285
7286     my $res = [];
7287     foreach my $id (keys %$data) {
7288         foreach my $item (@{$data->{$id}}) {
7289             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
7290             push @$res, $item->{volid} if defined($item->{volid});
7291         }
7292     }
7293
7294     return $res;
7295 }
7296
7297 my $complete_vmid_full = sub {
7298     my ($running) = @_;
7299
7300     my $idlist = vmstatus();
7301
7302     my $res = [];
7303
7304     foreach my $id (keys %$idlist) {
7305         my $d = $idlist->{$id};
7306         if (defined($running)) {
7307             next if $d->{template};
7308             next if $running && $d->{status} ne 'running';
7309             next if !$running && $d->{status} eq 'running';
7310         }
7311         push @$res, $id;
7312
7313     }
7314     return $res;
7315 };
7316
7317 sub complete_vmid {
7318     return &$complete_vmid_full();
7319 }
7320
7321 sub complete_vmid_stopped {
7322     return &$complete_vmid_full(0);
7323 }
7324
7325 sub complete_vmid_running {
7326     return &$complete_vmid_full(1);
7327 }
7328
7329 sub complete_storage {
7330
7331     my $cfg = PVE::Storage::config();
7332     my $ids = $cfg->{ids};
7333
7334     my $res = [];
7335     foreach my $sid (keys %$ids) {
7336         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7337         next if !$ids->{$sid}->{content}->{images};
7338         push @$res, $sid;
7339     }
7340
7341     return $res;
7342 }
7343
7344 sub complete_migration_storage {
7345     my ($cmd, $param, $current_value, $all_args) = @_;
7346
7347     my $targetnode = @$all_args[1];
7348
7349     my $cfg = PVE::Storage::config();
7350     my $ids = $cfg->{ids};
7351
7352     my $res = [];
7353     foreach my $sid (keys %$ids) {
7354         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7355         next if !$ids->{$sid}->{content}->{images};
7356         push @$res, $sid;
7357     }
7358
7359     return $res;
7360 }
7361
7362 1;