PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::DataCenterConfig;
  31 use PVE::Exception qw(raise raise_param_exc);
  32 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  33 use PVE::INotify;
  34 use PVE::JSONSchema qw(get_standard_option);
  35 use PVE::ProcFSTools;
  36 use PVE::RPCEnvironment;
  37 use PVE::Storage;
  38 use PVE::SysFSTools;
  39 use PVE::Systemd;
  40 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  41
  42 use PVE::QMPClient;
  43 use PVE::QemuConfig;
  44 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  45 use PVE::QemuServer::Cloudinit;
  46 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  47 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom parse_drive print_drive);
  48 use PVE::QemuServer::Machine;
  49 use PVE::QemuServer::Memory;
  50 use PVE::QemuServer::Monitor qw(mon_cmd);
  51 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  52 use PVE::QemuServer::USB qw(parse_usb_device);
  53
  54 my $have_sdn;
  55 eval {
  56     require PVE::Network::SDN::Zones;
  57     $have_sdn = 1;
  58 };
  59
  60 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  61 my $OVMF = {
  62     x86_64 => [
  63         "$EDK2_FW_BASE/OVMF_CODE.fd",
  64         "$EDK2_FW_BASE/OVMF_VARS.fd"
  65     ],
  66     aarch64 => [
  67         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  68         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  69     ],
  70 };
  71
  72 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  73
  74 # Note about locking: we use flock on the config file protect
  75 # against concurent actions.
  76 # Aditionaly, we have a 'lock' setting in the config file. This
  77 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  78 # allowed when such lock is set. But you can ignore this kind of
  79 # lock with the --skiplock flag.
  80
  81 cfs_register_file('/qemu-server/',
  82                   \&parse_vm_config,
  83                   \&write_vm_config);
  84
  85 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  86     description => "Some command save/restore state from this location.",
  87     type => 'string',
  88     maxLength => 128,
  89     optional => 1,
  90 });
  91
  92 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  93         description => "Specifies the Qemu machine type.",
  94         type => 'string',
  95         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
  96         maxLength => 40,
  97         optional => 1,
  98 });
  99
 100
 101 sub map_storage {
 102     my ($map, $source) = @_;
 103
 104     return $source if !defined($map);
 105
 106     return $map->{entries}->{$source}
 107         if $map->{entries} && defined($map->{entries}->{$source});
 108
 109     return $map->{default} if $map->{default};
 110
 111     # identity (fallback)
 112     return $source;
 113 }
 114
 115 PVE::JSONSchema::register_standard_option('pve-targetstorage', {
 116     description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
 117     type => 'string',
 118     format => 'storagepair-list',
 119     optional => 1,
 120 });
 121
 122 #no warnings 'redefine';
 123
 124 sub cgroups_write {
 125    my ($controller, $vmid, $option, $value) = @_;
 126
 127    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 128    PVE::ProcFSTools::write_proc_entry($path, $value);
 129
 130 }
 131
 132 my $nodename_cache;
 133 sub nodename {
 134     $nodename_cache //= PVE::INotify::nodename();
 135     return $nodename_cache;
 136 }
 137
 138 my $watchdog_fmt = {
 139     model => {
 140         default_key => 1,
 141         type => 'string',
 142         enum => [qw(i6300esb ib700)],
 143         description => "Watchdog type to emulate.",
 144         default => 'i6300esb',
 145         optional => 1,
 146     },
 147     action => {
 148         type => 'string',
 149         enum => [qw(reset shutdown poweroff pause debug none)],
 150         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 151         optional => 1,
 152     },
 153 };
 154 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 155
 156 my $agent_fmt = {
 157     enabled => {
 158         description => "Enable/disable Qemu GuestAgent.",
 159         type => 'boolean',
 160         default => 0,
 161         default_key => 1,
 162     },
 163     fstrim_cloned_disks => {
 164         description => "Run fstrim after cloning/moving a disk.",
 165         type => 'boolean',
 166         optional => 1,
 167         default => 0
 168     },
 169     type => {
 170         description => "Select the agent type",
 171         type => 'string',
 172         default => 'virtio',
 173         optional => 1,
 174         enum => [qw(virtio isa)],
 175     },
 176 };
 177
 178 my $vga_fmt = {
 179     type => {
 180         description => "Select the VGA type.",
 181         type => 'string',
 182         default => 'std',
 183         optional => 1,
 184         default_key => 1,
 185         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 186     },
 187     memory => {
 188         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 189         type => 'integer',
 190         optional => 1,
 191         minimum => 4,
 192         maximum => 512,
 193     },
 194 };
 195
 196 my $ivshmem_fmt = {
 197     size => {
 198         type => 'integer',
 199         minimum => 1,
 200         description => "The size of the file in MB.",
 201     },
 202     name => {
 203         type => 'string',
 204         pattern => '[a-zA-Z0-9\-]+',
 205         optional => 1,
 206         format_description => 'string',
 207         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 208     },
 209 };
 210
 211 my $audio_fmt = {
 212     device => {
 213         type => 'string',
 214         enum => [qw(ich9-intel-hda intel-hda AC97)],
 215         description =>  "Configure an audio device."
 216     },
 217     driver =>  {
 218         type => 'string',
 219         enum => ['spice'],
 220         default => 'spice',
 221         optional => 1,
 222         description => "Driver backend for the audio device."
 223     },
 224 };
 225
 226 my $spice_enhancements_fmt = {
 227     foldersharing => {
 228         type => 'boolean',
 229         optional => 1,
 230         default => '0',
 231         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 232     },
 233     videostreaming =>  {
 234         type => 'string',
 235         enum => ['off', 'all', 'filter'],
 236         default => 'off',
 237         optional => 1,
 238         description => "Enable video streaming. Uses compression for detected video streams."
 239     },
 240 };
 241
 242 my $rng_fmt = {
 243     source => {
 244         type => 'string',
 245         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 246         default_key => 1,
 247         description => "The file on the host to gather entropy from. In most"
 248                      . " cases /dev/urandom should be preferred over /dev/random"
 249                      . " to avoid entropy-starvation issues on the host. Using"
 250                      . " urandom does *not* decrease security in any meaningful"
 251                      . " way, as it's still seeded from real entropy, and the"
 252                      . " bytes provided will most likely be mixed with real"
 253                      . " entropy on the guest as well. /dev/hwrng can be used"
 254                      . " to pass through a hardware RNG from the host.",
 255     },
 256     max_bytes => {
 257         type => 'integer',
 258         description => "Maximum bytes of entropy injected into the guest every"
 259                      . " 'period' milliseconds. Prefer a lower value when using"
 260                      . " /dev/random as source. Use 0 to disable limiting"
 261                      . " (potentially dangerous!).",
 262         optional => 1,
 263
 264         # default is 1 KiB/s, provides enough entropy to the guest to avoid
 265         # boot-starvation issues (e.g. systemd etc...) while allowing no chance
 266         # of overwhelming the host, provided we're reading from /dev/urandom
 267         default => 1024,
 268     },
 269     period => {
 270         type => 'integer',
 271         description => "Every 'period' milliseconds the entropy-injection quota"
 272                      . " is reset, allowing the guest to retrieve another"
 273                      . " 'max_bytes' of entropy.",
 274         optional => 1,
 275         default => 1000,
 276     },
 277 };
 278
 279 my $confdesc = {
 280     onboot => {
 281         optional => 1,
 282         type => 'boolean',
 283         description => "Specifies whether a VM will be started during system bootup.",
 284         default => 0,
 285     },
 286     autostart => {
 287         optional => 1,
 288         type => 'boolean',
 289         description => "Automatic restart after crash (currently ignored).",
 290         default => 0,
 291     },
 292     hotplug => {
 293         optional => 1,
 294         type => 'string', format => 'pve-hotplug-features',
 295         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 296         default => 'network,disk,usb',
 297     },
 298     reboot => {
 299         optional => 1,
 300         type => 'boolean',
 301         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 302         default => 1,
 303     },
 304     lock => {
 305         optional => 1,
 306         type => 'string',
 307         description => "Lock/unlock the VM.",
 308         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 309     },
 310     cpulimit => {
 311         optional => 1,
 312         type => 'number',
 313         description => "Limit of CPU usage.",
 314         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 315         minimum => 0,
 316         maximum => 128,
 317         default => 0,
 318     },
 319     cpuunits => {
 320         optional => 1,
 321         type => 'integer',
 322         description => "CPU weight for a VM.",
 323         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 324         minimum => 2,
 325         maximum => 262144,
 326         default => 1024,
 327     },
 328     memory => {
 329         optional => 1,
 330         type => 'integer',
 331         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 332         minimum => 16,
 333         default => 512,
 334     },
 335     balloon => {
 336         optional => 1,
 337         type => 'integer',
 338         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 339         minimum => 0,
 340     },
 341     shares => {
 342         optional => 1,
 343         type => 'integer',
 344         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 345         minimum => 0,
 346         maximum => 50000,
 347         default => 1000,
 348     },
 349     keyboard => {
 350         optional => 1,
 351         type => 'string',
 352         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 353                        "It should not be necessary to set it.",
 354         enum => PVE::Tools::kvmkeymaplist(),
 355         default => undef,
 356     },
 357     name => {
 358         optional => 1,
 359         type => 'string', format => 'dns-name',
 360         description => "Set a name for the VM. Only used on the configuration web interface.",
 361     },
 362     scsihw => {
 363         optional => 1,
 364         type => 'string',
 365         description => "SCSI controller model",
 366         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 367         default => 'lsi',
 368     },
 369     description => {
 370         optional => 1,
 371         type => 'string',
 372         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 373     },
 374     ostype => {
 375         optional => 1,
 376         type => 'string',
 377         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 378         description => "Specify guest operating system.",
 379         verbose_description => <<EODESC,
 380 Specify guest operating system. This is used to enable special
 381 optimization/features for specific operating systems:
 382
 383 [horizontal]
 384 other;; unspecified OS
 385 wxp;; Microsoft Windows XP
 386 w2k;; Microsoft Windows 2000
 387 w2k3;; Microsoft Windows 2003
 388 w2k8;; Microsoft Windows 2008
 389 wvista;; Microsoft Windows Vista
 390 win7;; Microsoft Windows 7
 391 win8;; Microsoft Windows 8/2012/2012r2
 392 win10;; Microsoft Windows 10/2016
 393 l24;; Linux 2.4 Kernel
 394 l26;; Linux 2.6 - 5.X Kernel
 395 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 396 EODESC
 397     },
 398     boot => {
 399         optional => 1,
 400         type => 'string',
 401         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 402         pattern => '[acdn]{1,4}',
 403         default => 'cdn',
 404     },
 405     bootdisk => {
 406         optional => 1,
 407         type => 'string', format => 'pve-qm-bootdisk',
 408         description => "Enable booting from specified disk.",
 409         pattern => '(ide|sata|scsi|virtio)\d+',
 410     },
 411     smp => {
 412         optional => 1,
 413         type => 'integer',
 414         description => "The number of CPUs. Please use option -sockets instead.",
 415         minimum => 1,
 416         default => 1,
 417     },
 418     sockets => {
 419         optional => 1,
 420         type => 'integer',
 421         description => "The number of CPU sockets.",
 422         minimum => 1,
 423         default => 1,
 424     },
 425     cores => {
 426         optional => 1,
 427         type => 'integer',
 428         description => "The number of cores per socket.",
 429         minimum => 1,
 430         default => 1,
 431     },
 432     numa => {
 433         optional => 1,
 434         type => 'boolean',
 435         description => "Enable/disable NUMA.",
 436         default => 0,
 437     },
 438     hugepages => {
 439         optional => 1,
 440         type => 'string',
 441         description => "Enable/disable hugepages memory.",
 442         enum => [qw(any 2 1024)],
 443     },
 444     vcpus => {
 445         optional => 1,
 446         type => 'integer',
 447         description => "Number of hotplugged vcpus.",
 448         minimum => 1,
 449         default => 0,
 450     },
 451     acpi => {
 452         optional => 1,
 453         type => 'boolean',
 454         description => "Enable/disable ACPI.",
 455         default => 1,
 456     },
 457     agent => {
 458         optional => 1,
 459         description => "Enable/disable Qemu GuestAgent and its properties.",
 460         type => 'string',
 461         format => $agent_fmt,
 462     },
 463     kvm => {
 464         optional => 1,
 465         type => 'boolean',
 466         description => "Enable/disable KVM hardware virtualization.",
 467         default => 1,
 468     },
 469     tdf => {
 470         optional => 1,
 471         type => 'boolean',
 472         description => "Enable/disable time drift fix.",
 473         default => 0,
 474     },
 475     localtime => {
 476         optional => 1,
 477         type => 'boolean',
 478         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 479     },
 480     freeze => {
 481         optional => 1,
 482         type => 'boolean',
 483         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 484     },
 485     vga => {
 486         optional => 1,
 487         type => 'string', format => $vga_fmt,
 488         description => "Configure the VGA hardware.",
 489         verbose_description => "Configure the VGA Hardware. If you want to use ".
 490             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 491             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 492             "is 'std' for all OS types besides some Windows versions (XP and " .
 493             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 494             "display server. For win* OS you can select how many independent " .
 495             "displays you want, Linux guests can add displays them self.\n".
 496             "You can also run without any graphic card, using a serial device as terminal.",
 497     },
 498     watchdog => {
 499         optional => 1,
 500         type => 'string', format => 'pve-qm-watchdog',
 501         description => "Create a virtual hardware watchdog device.",
 502         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 503             " (by a guest action), the watchdog must be periodically polled " .
 504             "by an agent inside the guest or else the watchdog will reset " .
 505             "the guest (or execute the respective action specified)",
 506     },
 507     startdate => {
 508         optional => 1,
 509         type => 'string',
 510         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 511         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 512         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 513         default => 'now',
 514     },
 515     startup =>  get_standard_option('pve-startup-order'),
 516     template => {
 517         optional => 1,
 518         type => 'boolean',
 519         description => "Enable/disable Template.",
 520         default => 0,
 521     },
 522     args => {
 523         optional => 1,
 524         type => 'string',
 525         description => "Arbitrary arguments passed to kvm.",
 526         verbose_description => <<EODESCR,
 527 Arbitrary arguments passed to kvm, for example:
 528
 529 args: -no-reboot -no-hpet
 530
 531 NOTE: this option is for experts only.
 532 EODESCR
 533     },
 534     tablet => {
 535         optional => 1,
 536         type => 'boolean',
 537         default => 1,
 538         description => "Enable/disable the USB tablet device.",
 539         verbose_description => "Enable/disable the USB tablet device. This device is " .
 540             "usually needed to allow absolute mouse positioning with VNC. " .
 541             "Else the mouse runs out of sync with normal VNC clients. " .
 542             "If you're running lots of console-only guests on one host, " .
 543             "you may consider disabling this to save some context switches. " .
 544             "This is turned off by default if you use spice (-vga=qxl).",
 545     },
 546     migrate_speed => {
 547         optional => 1,
 548         type => 'integer',
 549         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 550         minimum => 0,
 551         default => 0,
 552     },
 553     migrate_downtime => {
 554         optional => 1,
 555         type => 'number',
 556         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 557         minimum => 0,
 558         default => 0.1,
 559     },
 560     cdrom => {
 561         optional => 1,
 562         type => 'string', format => 'pve-qm-ide',
 563         typetext => '<volume>',
 564         description => "This is an alias for option -ide2",
 565     },
 566     cpu => {
 567         optional => 1,
 568         description => "Emulated CPU type.",
 569         type => 'string',
 570         format => 'pve-vm-cpu-conf',
 571     },
 572     parent => get_standard_option('pve-snapshot-name', {
 573         optional => 1,
 574         description => "Parent snapshot name. This is used internally, and should not be modified.",
 575     }),
 576     snaptime => {
 577         optional => 1,
 578         description => "Timestamp for snapshots.",
 579         type => 'integer',
 580         minimum => 0,
 581     },
 582     vmstate => {
 583         optional => 1,
 584         type => 'string', format => 'pve-volume-id',
 585         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 586     },
 587     vmstatestorage => get_standard_option('pve-storage-id', {
 588         description => "Default storage for VM state volumes/files.",
 589         optional => 1,
 590     }),
 591     runningmachine => get_standard_option('pve-qemu-machine', {
 592         description => "Specifies the QEMU machine type of the running vm. This is used internally for snapshots.",
 593     }),
 594     runningcpu => {
 595         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used internally for snapshots.",
 596         optional => 1,
 597         type => 'string',
 598         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 599         format_description => 'QEMU -cpu parameter'
 600     },
 601     machine => get_standard_option('pve-qemu-machine'),
 602     arch => {
 603         description => "Virtual processor architecture. Defaults to the host.",
 604         optional => 1,
 605         type => 'string',
 606         enum => [qw(x86_64 aarch64)],
 607     },
 608     smbios1 => {
 609         description => "Specify SMBIOS type 1 fields.",
 610         type => 'string', format => 'pve-qm-smbios1',
 611         maxLength => 512,
 612         optional => 1,
 613     },
 614     protection => {
 615         optional => 1,
 616         type => 'boolean',
 617         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 618         default => 0,
 619     },
 620     bios => {
 621         optional => 1,
 622         type => 'string',
 623         enum => [ qw(seabios ovmf) ],
 624         description => "Select BIOS implementation.",
 625         default => 'seabios',
 626     },
 627     vmgenid => {
 628         type => 'string',
 629         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 630         format_description => 'UUID',
 631         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 632         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 633             " 128-bit integer value identifier to the guest OS. This allows to".
 634             " notify the guest operating system when the virtual machine is".
 635             " executed with a different configuration (e.g. snapshot execution".
 636             " or creation from a template). The guest operating system notices".
 637             " the change, and is then able to react as appropriate by marking".
 638             " its copies of distributed databases as dirty, re-initializing its".
 639             " random number generator, etc.\n".
 640             "Note that auto-creation only works when done throug API/CLI create".
 641             " or update methods, but not when manually editing the config file.",
 642         default => "1 (autogenerated)",
 643         optional => 1,
 644     },
 645     hookscript => {
 646         type => 'string',
 647         format => 'pve-volume-id',
 648         optional => 1,
 649         description => "Script that will be executed during various steps in the vms lifetime.",
 650     },
 651     ivshmem => {
 652         type => 'string',
 653         format => $ivshmem_fmt,
 654         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 655         optional => 1,
 656     },
 657     audio0 => {
 658         type => 'string',
 659         format => $audio_fmt,
 660         description => "Configure a audio device, useful in combination with QXL/Spice.",
 661         optional => 1
 662     },
 663     spice_enhancements => {
 664         type => 'string',
 665         format => $spice_enhancements_fmt,
 666         description => "Configure additional enhancements for SPICE.",
 667         optional => 1
 668     },
 669     tags => {
 670         type => 'string', format => 'pve-tag-list',
 671         description => 'Tags of the VM. This is only meta information.',
 672         optional => 1,
 673     },
 674     rng0 => {
 675         type => 'string',
 676         format => $rng_fmt,
 677         description => "Configure a VirtIO-based Random Number Generator.",
 678         optional => 1,
 679     },
 680 };
 681
 682 my $cicustom_fmt = {
 683     meta => {
 684         type => 'string',
 685         optional => 1,
 686         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 687         format => 'pve-volume-id',
 688         format_description => 'volume',
 689     },
 690     network => {
 691         type => 'string',
 692         optional => 1,
 693         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 694         format => 'pve-volume-id',
 695         format_description => 'volume',
 696     },
 697     user => {
 698         type => 'string',
 699         optional => 1,
 700         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 701         format => 'pve-volume-id',
 702         format_description => 'volume',
 703     },
 704 };
 705 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 706
 707 my $confdesc_cloudinit = {
 708     citype => {
 709         optional => 1,
 710         type => 'string',
 711         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 712         enum => ['configdrive2', 'nocloud'],
 713     },
 714     ciuser => {
 715         optional => 1,
 716         type => 'string',
 717         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 718     },
 719     cipassword => {
 720         optional => 1,
 721         type => 'string',
 722         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 723     },
 724     cicustom => {
 725         optional => 1,
 726         type => 'string',
 727         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 728         format => 'pve-qm-cicustom',
 729     },
 730     searchdomain => {
 731         optional => 1,
 732         type => 'string',
 733         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 734     },
 735     nameserver => {
 736         optional => 1,
 737         type => 'string', format => 'address-list',
 738         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 739     },
 740     sshkeys => {
 741         optional => 1,
 742         type => 'string',
 743         format => 'urlencoded',
 744         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 745     },
 746 };
 747
 748 # what about other qemu settings ?
 749 #cpu => 'string',
 750 #machine => 'string',
 751 #fda => 'file',
 752 #fdb => 'file',
 753 #mtdblock => 'file',
 754 #sd => 'file',
 755 #pflash => 'file',
 756 #snapshot => 'bool',
 757 #bootp => 'file',
 758 ##tftp => 'dir',
 759 ##smb => 'dir',
 760 #kernel => 'file',
 761 #append => 'string',
 762 #initrd => 'file',
 763 ##soundhw => 'string',
 764
 765 while (my ($k, $v) = each %$confdesc) {
 766     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 767 }
 768
 769 my $MAX_USB_DEVICES = 5;
 770 my $MAX_NETS = 32;
 771 my $MAX_SERIAL_PORTS = 4;
 772 my $MAX_PARALLEL_PORTS = 3;
 773 my $MAX_NUMA = 8;
 774
 775 my $numa_fmt = {
 776     cpus => {
 777         type => "string",
 778         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 779         description => "CPUs accessing this NUMA node.",
 780         format_description => "id[-id];...",
 781     },
 782     memory => {
 783         type => "number",
 784         description => "Amount of memory this NUMA node provides.",
 785         optional => 1,
 786     },
 787     hostnodes => {
 788         type => "string",
 789         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 790         description => "Host NUMA nodes to use.",
 791         format_description => "id[-id];...",
 792         optional => 1,
 793     },
 794     policy => {
 795         type => 'string',
 796         enum => [qw(preferred bind interleave)],
 797         description => "NUMA allocation policy.",
 798         optional => 1,
 799     },
 800 };
 801 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 802 my $numadesc = {
 803     optional => 1,
 804     type => 'string', format => $numa_fmt,
 805     description => "NUMA topology.",
 806 };
 807 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 808
 809 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 810     $confdesc->{"numa$i"} = $numadesc;
 811 }
 812
 813 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 814                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 815                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 816 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 817
 818 my $net_fmt_bridge_descr = <<__EOD__;
 819 Bridge to attach the network device to. The Proxmox VE standard bridge
 820 is called 'vmbr0'.
 821
 822 If you do not specify a bridge, we create a kvm user (NATed) network
 823 device, which provides DHCP and DNS services. The following addresses
 824 are used:
 825
 826  10.0.2.2   Gateway
 827  10.0.2.3   DNS Server
 828  10.0.2.4   SMB Server
 829
 830 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 831 __EOD__
 832
 833 my $net_fmt = {
 834     macaddr  => get_standard_option('mac-addr', {
 835         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 836     }),
 837     model => {
 838         type => 'string',
 839         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 840         enum => $nic_model_list,
 841         default_key => 1,
 842     },
 843     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 844     bridge => {
 845         type => 'string',
 846         description => $net_fmt_bridge_descr,
 847         format_description => 'bridge',
 848         pattern => '[-_.\w\d]+',
 849         optional => 1,
 850     },
 851     queues => {
 852         type => 'integer',
 853         minimum => 0, maximum => 16,
 854         description => 'Number of packet queues to be used on the device.',
 855         optional => 1,
 856     },
 857     rate => {
 858         type => 'number',
 859         minimum => 0,
 860         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 861         optional => 1,
 862     },
 863     tag => {
 864         type => 'integer',
 865         minimum => 1, maximum => 4094,
 866         description => 'VLAN tag to apply to packets on this interface.',
 867         optional => 1,
 868     },
 869     trunks => {
 870         type => 'string',
 871         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 872         description => 'VLAN trunks to pass through this interface.',
 873         format_description => 'vlanid[;vlanid...]',
 874         optional => 1,
 875     },
 876     firewall => {
 877         type => 'boolean',
 878         description => 'Whether this interface should be protected by the firewall.',
 879         optional => 1,
 880     },
 881     link_down => {
 882         type => 'boolean',
 883         description => 'Whether this interface should be disconnected (like pulling the plug).',
 884         optional => 1,
 885     },
 886     mtu => {
 887         type => 'integer',
 888         minimum => 1, maximum => 65520,
 889         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 890         optional => 1,
 891     },
 892 };
 893
 894 my $netdesc = {
 895     optional => 1,
 896     type => 'string', format => $net_fmt,
 897     description => "Specify network devices.",
 898 };
 899
 900 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 901
 902 my $ipconfig_fmt = {
 903     ip => {
 904         type => 'string',
 905         format => 'pve-ipv4-config',
 906         format_description => 'IPv4Format/CIDR',
 907         description => 'IPv4 address in CIDR format.',
 908         optional => 1,
 909         default => 'dhcp',
 910     },
 911     gw => {
 912         type => 'string',
 913         format => 'ipv4',
 914         format_description => 'GatewayIPv4',
 915         description => 'Default gateway for IPv4 traffic.',
 916         optional => 1,
 917         requires => 'ip',
 918     },
 919     ip6 => {
 920         type => 'string',
 921         format => 'pve-ipv6-config',
 922         format_description => 'IPv6Format/CIDR',
 923         description => 'IPv6 address in CIDR format.',
 924         optional => 1,
 925         default => 'dhcp',
 926     },
 927     gw6 => {
 928         type => 'string',
 929         format => 'ipv6',
 930         format_description => 'GatewayIPv6',
 931         description => 'Default gateway for IPv6 traffic.',
 932         optional => 1,
 933         requires => 'ip6',
 934     },
 935 };
 936 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 937 my $ipconfigdesc = {
 938     optional => 1,
 939     type => 'string', format => 'pve-qm-ipconfig',
 940     description => <<'EODESCR',
 941 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 942
 943 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 944
 945 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 946 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 947
 948 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 949 EODESCR
 950 };
 951 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 952
 953 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 954     $confdesc->{"net$i"} = $netdesc;
 955     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 956 }
 957
 958 foreach my $key (keys %$confdesc_cloudinit) {
 959     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 960 }
 961
 962 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 963 sub verify_volume_id_or_qm_path {
 964     my ($volid, $noerr) = @_;
 965
 966     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 967         return $volid;
 968     }
 969
 970     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 971     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 972     if ($@) {
 973         return undef if $noerr;
 974         die $@;
 975     }
 976     return $volid;
 977 }
 978
 979 my $usb_fmt = {
 980     host => {
 981         default_key => 1,
 982         type => 'string', format => 'pve-qm-usb-device',
 983         format_description => 'HOSTUSBDEVICE|spice',
 984         description => <<EODESCR,
 985 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
 986
 987  'bus-port(.port)*' (decimal numbers) or
 988  'vendor_id:product_id' (hexadeciaml numbers) or
 989  'spice'
 990
 991 You can use the 'lsusb -t' command to list existing usb devices.
 992
 993 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 994
 995 The value 'spice' can be used to add a usb redirection devices for spice.
 996 EODESCR
 997     },
 998     usb3 => {
 999         optional => 1,
1000         type => 'boolean',
1001         description => "Specifies whether if given host option is a USB3 device or port.",
1002         default => 0,
1003     },
1004 };
1005
1006 my $usbdesc = {
1007     optional => 1,
1008     type => 'string', format => $usb_fmt,
1009     description => "Configure an USB device (n is 0 to 4).",
1010 };
1011 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1012
1013 my $serialdesc = {
1014         optional => 1,
1015         type => 'string',
1016         pattern => '(/dev/.+|socket)',
1017         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1018         verbose_description =>  <<EODESCR,
1019 Create a serial device inside the VM (n is 0 to 3), and pass through a
1020 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1021 host side (use 'qm terminal' to open a terminal connection).
1022
1023 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1024
1025 CAUTION: Experimental! User reported problems with this option.
1026 EODESCR
1027 };
1028
1029 my $paralleldesc= {
1030         optional => 1,
1031         type => 'string',
1032         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1033         description =>  "Map host parallel devices (n is 0 to 2).",
1034         verbose_description =>  <<EODESCR,
1035 Map host parallel devices (n is 0 to 2).
1036
1037 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1038
1039 CAUTION: Experimental! User reported problems with this option.
1040 EODESCR
1041 };
1042
1043 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1044     $confdesc->{"parallel$i"} = $paralleldesc;
1045 }
1046
1047 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1048     $confdesc->{"serial$i"} = $serialdesc;
1049 }
1050
1051 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1052     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1053 }
1054
1055 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1056     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1057 }
1058
1059 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1060     $confdesc->{"usb$i"} = $usbdesc;
1061 }
1062
1063 my $kvm_api_version = 0;
1064
1065 sub kvm_version {
1066     return $kvm_api_version if $kvm_api_version;
1067
1068     open my $fh, '<', '/dev/kvm'
1069         or return undef;
1070
1071     # 0xae00 => KVM_GET_API_VERSION
1072     $kvm_api_version = ioctl($fh, 0xae00, 0);
1073
1074     return $kvm_api_version;
1075 }
1076
1077 my $kvm_user_version = {};
1078 my $kvm_mtime = {};
1079
1080 sub kvm_user_version {
1081     my ($binary) = @_;
1082
1083     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1084     my $st = stat($binary);
1085
1086     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1087     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1088         $cachedmtime == $st->mtime;
1089
1090     $kvm_user_version->{$binary} = 'unknown';
1091     $kvm_mtime->{$binary} = $st->mtime;
1092
1093     my $code = sub {
1094         my $line = shift;
1095         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1096             $kvm_user_version->{$binary} = $2;
1097         }
1098     };
1099
1100     eval { run_command([$binary, '--version'], outfunc => $code); };
1101     warn $@ if $@;
1102
1103     return $kvm_user_version->{$binary};
1104
1105 }
1106
1107 sub kernel_has_vhost_net {
1108     return -c '/dev/vhost-net';
1109 }
1110
1111 sub option_exists {
1112     my $key = shift;
1113     return defined($confdesc->{$key});
1114 }
1115
1116 my $cdrom_path;
1117 sub get_cdrom_path {
1118
1119     return  $cdrom_path if $cdrom_path;
1120
1121     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1122     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1123     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1124 }
1125
1126 sub get_iso_path {
1127     my ($storecfg, $vmid, $cdrom) = @_;
1128
1129     if ($cdrom eq 'cdrom') {
1130         return get_cdrom_path();
1131     } elsif ($cdrom eq 'none') {
1132         return '';
1133     } elsif ($cdrom =~ m|^/|) {
1134         return $cdrom;
1135     } else {
1136         return PVE::Storage::path($storecfg, $cdrom);
1137     }
1138 }
1139
1140 # try to convert old style file names to volume IDs
1141 sub filename_to_volume_id {
1142     my ($vmid, $file, $media) = @_;
1143
1144      if (!($file eq 'none' || $file eq 'cdrom' ||
1145           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1146
1147         return undef if $file =~ m|/|;
1148
1149         if ($media && $media eq 'cdrom') {
1150             $file = "local:iso/$file";
1151         } else {
1152             $file = "local:$vmid/$file";
1153         }
1154     }
1155
1156     return $file;
1157 }
1158
1159 sub verify_media_type {
1160     my ($opt, $vtype, $media) = @_;
1161
1162     return if !$media;
1163
1164     my $etype;
1165     if ($media eq 'disk') {
1166         $etype = 'images';
1167     } elsif ($media eq 'cdrom') {
1168         $etype = 'iso';
1169     } else {
1170         die "internal error";
1171     }
1172
1173     return if ($vtype eq $etype);
1174
1175     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1176 }
1177
1178 sub cleanup_drive_path {
1179     my ($opt, $storecfg, $drive) = @_;
1180
1181     # try to convert filesystem paths to volume IDs
1182
1183     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1184         ($drive->{file} !~ m|^/dev/.+|) &&
1185         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1186         ($drive->{file} !~ m/^\d+$/)) {
1187         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1188         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1189         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1190         verify_media_type($opt, $vtype, $drive->{media});
1191         $drive->{file} = $volid;
1192     }
1193
1194     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1195 }
1196
1197 sub parse_hotplug_features {
1198     my ($data) = @_;
1199
1200     my $res = {};
1201
1202     return $res if $data eq '0';
1203
1204     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1205
1206     foreach my $feature (PVE::Tools::split_list($data)) {
1207         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1208             $res->{$1} = 1;
1209         } else {
1210             die "invalid hotplug feature '$feature'\n";
1211         }
1212     }
1213     return $res;
1214 }
1215
1216 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1217 sub pve_verify_hotplug_features {
1218     my ($value, $noerr) = @_;
1219
1220     return $value if parse_hotplug_features($value);
1221
1222     return undef if $noerr;
1223
1224     die "unable to parse hotplug option\n";
1225 }
1226
1227 sub scsi_inquiry {
1228     my($fh, $noerr) = @_;
1229
1230     my $SG_IO = 0x2285;
1231     my $SG_GET_VERSION_NUM = 0x2282;
1232
1233     my $versionbuf = "\x00" x 8;
1234     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1235     if (!$ret) {
1236         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1237         return undef;
1238     }
1239     my $version = unpack("I", $versionbuf);
1240     if ($version < 30000) {
1241         die "scsi generic interface too old\n"  if !$noerr;
1242         return undef;
1243     }
1244
1245     my $buf = "\x00" x 36;
1246     my $sensebuf = "\x00" x 8;
1247     my $cmd = pack("C x3 C x1", 0x12, 36);
1248
1249     # see /usr/include/scsi/sg.h
1250     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1251
1252     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1253                       length($sensebuf), 0, length($buf), $buf,
1254                       $cmd, $sensebuf, 6000);
1255
1256     $ret = ioctl($fh, $SG_IO, $packet);
1257     if (!$ret) {
1258         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1259         return undef;
1260     }
1261
1262     my @res = unpack($sg_io_hdr_t, $packet);
1263     if ($res[17] || $res[18]) {
1264         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1265         return undef;
1266     }
1267
1268     my $res = {};
1269     (my $byte0, my $byte1, $res->{vendor},
1270      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1271
1272     $res->{removable} = $byte1 & 128 ? 1 : 0;
1273     $res->{type} = $byte0 & 31;
1274
1275     return $res;
1276 }
1277
1278 sub path_is_scsi {
1279     my ($path) = @_;
1280
1281     my $fh = IO::File->new("+<$path") || return undef;
1282     my $res = scsi_inquiry($fh, 1);
1283     close($fh);
1284
1285     return $res;
1286 }
1287
1288 sub print_tabletdevice_full {
1289     my ($conf, $arch) = @_;
1290
1291     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1292
1293     # we use uhci for old VMs because tablet driver was buggy in older qemu
1294     my $usbbus;
1295     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1296         $usbbus = 'ehci';
1297     } else {
1298         $usbbus = 'uhci';
1299     }
1300
1301     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1302 }
1303
1304 sub print_keyboarddevice_full {
1305     my ($conf, $arch, $machine) = @_;
1306
1307     return undef if $arch ne 'aarch64';
1308
1309     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1310 }
1311
1312 sub print_drivedevice_full {
1313     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1314
1315     my $device = '';
1316     my $maxdev = 0;
1317
1318     if ($drive->{interface} eq 'virtio') {
1319         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1320         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1321         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1322     } elsif ($drive->{interface} eq 'scsi') {
1323
1324         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1325         my $unit = $drive->{index} % $maxdev;
1326         my $devicetype = 'hd';
1327         my $path = '';
1328         if (drive_is_cdrom($drive)) {
1329             $devicetype = 'cd';
1330         } else {
1331             if ($drive->{file} =~ m|^/|) {
1332                 $path = $drive->{file};
1333                 if (my $info = path_is_scsi($path)) {
1334                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1335                         $devicetype = 'block';
1336                     } elsif ($info->{type} == 1) { # tape
1337                         $devicetype = 'generic';
1338                     }
1339                 }
1340             } else {
1341                  $path = PVE::Storage::path($storecfg, $drive->{file});
1342             }
1343
1344             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1345             my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
1346             if ($path =~ m/^iscsi\:\/\// &&
1347                !min_version($version, 4, 1)) {
1348                 $devicetype = 'generic';
1349             }
1350         }
1351
1352         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1353             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1354         } else {
1355             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1356         }
1357
1358         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1359             $device .= ",rotation_rate=1";
1360         }
1361         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1362
1363     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1364         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1365         my $controller = int($drive->{index} / $maxdev);
1366         my $unit = $drive->{index} % $maxdev;
1367         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1368
1369         $device = "ide-$devicetype";
1370         if ($drive->{interface} eq 'ide') {
1371             $device .= ",bus=ide.$controller,unit=$unit";
1372         } else {
1373             $device .= ",bus=ahci$controller.$unit";
1374         }
1375         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1376
1377         if ($devicetype eq 'hd') {
1378             if (my $model = $drive->{model}) {
1379                 $model = URI::Escape::uri_unescape($model);
1380                 $device .= ",model=$model";
1381             }
1382             if ($drive->{ssd}) {
1383                 $device .= ",rotation_rate=1";
1384             }
1385         }
1386         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1387     } elsif ($drive->{interface} eq 'usb') {
1388         die "implement me";
1389         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1390     } else {
1391         die "unsupported interface type";
1392     }
1393
1394     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1395
1396     if (my $serial = $drive->{serial}) {
1397         $serial = URI::Escape::uri_unescape($serial);
1398         $device .= ",serial=$serial";
1399     }
1400
1401
1402     return $device;
1403 }
1404
1405 sub get_initiator_name {
1406     my $initiator;
1407
1408     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1409     while (defined(my $line = <$fh>)) {
1410         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1411         $initiator = $1;
1412         last;
1413     }
1414     $fh->close();
1415
1416     return $initiator;
1417 }
1418
1419 sub print_drive_commandline_full {
1420     my ($storecfg, $vmid, $drive) = @_;
1421
1422     my $path;
1423     my $volid = $drive->{file};
1424     my $format;
1425
1426     if (drive_is_cdrom($drive)) {
1427         $path = get_iso_path($storecfg, $vmid, $volid);
1428     } else {
1429         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1430         if ($storeid) {
1431             $path = PVE::Storage::path($storecfg, $volid);
1432             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1433             $format = qemu_img_format($scfg, $volname);
1434         } else {
1435             $path = $volid;
1436             $format = "raw";
1437         }
1438    }
1439
1440     my $opts = '';
1441     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1442     foreach my $o (@qemu_drive_options) {
1443         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1444     }
1445
1446     # snapshot only accepts on|off
1447     if (defined($drive->{snapshot})) {
1448         my $v = $drive->{snapshot} ? 'on' : 'off';
1449         $opts .= ",snapshot=$v";
1450     }
1451
1452     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1453         my ($dir, $qmpname) = @$type;
1454         if (my $v = $drive->{"mbps$dir"}) {
1455             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1456         }
1457         if (my $v = $drive->{"mbps${dir}_max"}) {
1458             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1459         }
1460         if (my $v = $drive->{"bps${dir}_max_length"}) {
1461             $opts .= ",throttling.bps$qmpname-max-length=$v";
1462         }
1463         if (my $v = $drive->{"iops${dir}"}) {
1464             $opts .= ",throttling.iops$qmpname=$v";
1465         }
1466         if (my $v = $drive->{"iops${dir}_max"}) {
1467             $opts .= ",throttling.iops$qmpname-max=$v";
1468         }
1469         if (my $v = $drive->{"iops${dir}_max_length"}) {
1470             $opts .= ",throttling.iops$qmpname-max-length=$v";
1471         }
1472     }
1473
1474     $opts .= ",format=$format" if $format && !$drive->{format};
1475
1476     my $cache_direct = 0;
1477
1478     if (my $cache = $drive->{cache}) {
1479         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1480     } elsif (!drive_is_cdrom($drive)) {
1481         $opts .= ",cache=none";
1482         $cache_direct = 1;
1483     }
1484
1485     # aio native works only with O_DIRECT
1486     if (!$drive->{aio}) {
1487         if($cache_direct) {
1488             $opts .= ",aio=native";
1489         } else {
1490             $opts .= ",aio=threads";
1491         }
1492     }
1493
1494     if (!drive_is_cdrom($drive)) {
1495         my $detectzeroes;
1496         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1497             $detectzeroes = 'off';
1498         } elsif ($drive->{discard}) {
1499             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1500         } else {
1501             # This used to be our default with discard not being specified:
1502             $detectzeroes = 'on';
1503         }
1504         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1505     }
1506
1507     my $pathinfo = $path ? "file=$path," : '';
1508
1509     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1510 }
1511
1512 sub print_netdevice_full {
1513     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1514
1515     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1516
1517     my $device = $net->{model};
1518     if ($net->{model} eq 'virtio') {
1519          $device = 'virtio-net-pci';
1520      };
1521
1522     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1523     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1524     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1525         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1526         my $vectors = $net->{queues} * 2 + 2;
1527         $tmpstr .= ",vectors=$vectors,mq=on";
1528     }
1529     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1530
1531     if (my $mtu = $net->{mtu}) {
1532         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1533             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1534             if ($mtu == 1) {
1535                  $mtu = $bridge_mtu;
1536             } elsif ($mtu < 576) {
1537                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1538             } elsif ($mtu > $bridge_mtu) {
1539                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1540             }
1541             $tmpstr .= ",host_mtu=$mtu";
1542         } else {
1543             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1544         }
1545     }
1546
1547     if ($use_old_bios_files) {
1548         my $romfile;
1549         if ($device eq 'virtio-net-pci') {
1550             $romfile = 'pxe-virtio.rom';
1551         } elsif ($device eq 'e1000') {
1552             $romfile = 'pxe-e1000.rom';
1553         } elsif ($device eq 'ne2k') {
1554             $romfile = 'pxe-ne2k_pci.rom';
1555         } elsif ($device eq 'pcnet') {
1556             $romfile = 'pxe-pcnet.rom';
1557         } elsif ($device eq 'rtl8139') {
1558             $romfile = 'pxe-rtl8139.rom';
1559         }
1560         $tmpstr .= ",romfile=$romfile" if $romfile;
1561     }
1562
1563     return $tmpstr;
1564 }
1565
1566 sub print_netdev_full {
1567     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1568
1569     my $i = '';
1570     if ($netid =~ m/^net(\d+)$/) {
1571         $i = int($1);
1572     }
1573
1574     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1575
1576     my $ifname = "tap${vmid}i$i";
1577
1578     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1579     die "interface name '$ifname' is too long (max 15 character)\n"
1580         if length($ifname) >= 16;
1581
1582     my $vhostparam = '';
1583     if (is_native($arch)) {
1584         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1585     }
1586
1587     my $vmname = $conf->{name} || "vm$vmid";
1588
1589     my $netdev = "";
1590     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1591
1592     if ($net->{bridge}) {
1593         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1594     } else {
1595         $netdev = "type=user,id=$netid,hostname=$vmname";
1596     }
1597
1598     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1599
1600     return $netdev;
1601 }
1602
1603 my $vga_map = {
1604     'cirrus' => 'cirrus-vga',
1605     'std' => 'VGA',
1606     'vmware' => 'vmware-svga',
1607     'virtio' => 'virtio-vga',
1608 };
1609
1610 sub print_vga_device {
1611     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1612
1613     my $type = $vga_map->{$vga->{type}};
1614     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1615         $type = 'virtio-gpu';
1616     }
1617     my $vgamem_mb = $vga->{memory};
1618
1619     my $max_outputs = '';
1620     if ($qxlnum) {
1621         $type = $id ? 'qxl' : 'qxl-vga';
1622
1623         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1624             # set max outputs so linux can have up to 4 qxl displays with one device
1625             if (min_version($machine_version, 4, 1)) {
1626                 $max_outputs = ",max_outputs=4";
1627             }
1628         }
1629     }
1630
1631     die "no devicetype for $vga->{type}\n" if !$type;
1632
1633     my $memory = "";
1634     if ($vgamem_mb) {
1635         if ($vga->{type} eq 'virtio') {
1636             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1637             $memory = ",max_hostmem=$bytes";
1638         } elsif ($qxlnum) {
1639             # from https://www.spice-space.org/multiple-monitors.html
1640             $memory = ",vgamem_mb=$vga->{memory}";
1641             my $ram = $vgamem_mb * 4;
1642             my $vram = $vgamem_mb * 2;
1643             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1644         } else {
1645             $memory = ",vgamem_mb=$vga->{memory}";
1646         }
1647     } elsif ($qxlnum && $id) {
1648         $memory = ",ram_size=67108864,vram_size=33554432";
1649     }
1650
1651     my $edidoff = "";
1652     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1653         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1654     }
1655
1656     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1657     my $vgaid = "vga" . ($id // '');
1658     my $pciaddr;
1659
1660     if ($q35 && $vgaid eq 'vga') {
1661         # the first display uses pcie.0 bus on q35 machines
1662         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1663     } else {
1664         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1665     }
1666
1667     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1668 }
1669
1670 sub parse_number_sets {
1671     my ($set) = @_;
1672     my $res = [];
1673     foreach my $part (split(/;/, $set)) {
1674         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1675             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1676             push @$res, [ $1, $2 ];
1677         } else {
1678             die "invalid range: $part\n";
1679         }
1680     }
1681     return $res;
1682 }
1683
1684 sub parse_numa {
1685     my ($data) = @_;
1686
1687     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1688     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1689     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1690     return $res;
1691 }
1692
1693 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1694 sub parse_net {
1695     my ($data) = @_;
1696
1697     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1698     if ($@) {
1699         warn $@;
1700         return undef;
1701     }
1702     if (!defined($res->{macaddr})) {
1703         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1704         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1705     }
1706     return $res;
1707 }
1708
1709 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1710 sub parse_ipconfig {
1711     my ($data) = @_;
1712
1713     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
1714     if ($@) {
1715         warn $@;
1716         return undef;
1717     }
1718
1719     if ($res->{gw} && !$res->{ip}) {
1720         warn 'gateway specified without specifying an IP address';
1721         return undef;
1722     }
1723     if ($res->{gw6} && !$res->{ip6}) {
1724         warn 'IPv6 gateway specified without specifying an IPv6 address';
1725         return undef;
1726     }
1727     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1728         warn 'gateway specified together with DHCP';
1729         return undef;
1730     }
1731     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1732         # gw6 + auto/dhcp
1733         warn "IPv6 gateway specified together with $res->{ip6} address";
1734         return undef;
1735     }
1736
1737     if (!$res->{ip} && !$res->{ip6}) {
1738         return { ip => 'dhcp', ip6 => 'dhcp' };
1739     }
1740
1741     return $res;
1742 }
1743
1744 sub print_net {
1745     my $net = shift;
1746
1747     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1748 }
1749
1750 sub add_random_macs {
1751     my ($settings) = @_;
1752
1753     foreach my $opt (keys %$settings) {
1754         next if $opt !~ m/^net(\d+)$/;
1755         my $net = parse_net($settings->{$opt});
1756         next if !$net;
1757         $settings->{$opt} = print_net($net);
1758     }
1759 }
1760
1761 sub vm_is_volid_owner {
1762     my ($storecfg, $vmid, $volid) = @_;
1763
1764     if ($volid !~  m|^/|) {
1765         my ($path, $owner);
1766         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1767         if ($owner && ($owner == $vmid)) {
1768             return 1;
1769         }
1770     }
1771
1772     return undef;
1773 }
1774
1775 sub vmconfig_register_unused_drive {
1776     my ($storecfg, $vmid, $conf, $drive) = @_;
1777
1778     if (drive_is_cloudinit($drive)) {
1779         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1780         warn $@ if $@;
1781     } elsif (!drive_is_cdrom($drive)) {
1782         my $volid = $drive->{file};
1783         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1784             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1785         }
1786     }
1787 }
1788
1789 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1790 my $smbios1_fmt = {
1791     uuid => {
1792         type => 'string',
1793         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1794         format_description => 'UUID',
1795         description => "Set SMBIOS1 UUID.",
1796         optional => 1,
1797     },
1798     version => {
1799         type => 'string',
1800         pattern => '[A-Za-z0-9+\/]+={0,2}',
1801         format_description => 'Base64 encoded string',
1802         description => "Set SMBIOS1 version.",
1803         optional => 1,
1804     },
1805     serial => {
1806         type => 'string',
1807         pattern => '[A-Za-z0-9+\/]+={0,2}',
1808         format_description => 'Base64 encoded string',
1809         description => "Set SMBIOS1 serial number.",
1810         optional => 1,
1811     },
1812     manufacturer => {
1813         type => 'string',
1814         pattern => '[A-Za-z0-9+\/]+={0,2}',
1815         format_description => 'Base64 encoded string',
1816         description => "Set SMBIOS1 manufacturer.",
1817         optional => 1,
1818     },
1819     product => {
1820         type => 'string',
1821         pattern => '[A-Za-z0-9+\/]+={0,2}',
1822         format_description => 'Base64 encoded string',
1823         description => "Set SMBIOS1 product ID.",
1824         optional => 1,
1825     },
1826     sku => {
1827         type => 'string',
1828         pattern => '[A-Za-z0-9+\/]+={0,2}',
1829         format_description => 'Base64 encoded string',
1830         description => "Set SMBIOS1 SKU string.",
1831         optional => 1,
1832     },
1833     family => {
1834         type => 'string',
1835         pattern => '[A-Za-z0-9+\/]+={0,2}',
1836         format_description => 'Base64 encoded string',
1837         description => "Set SMBIOS1 family string.",
1838         optional => 1,
1839     },
1840     base64 => {
1841         type => 'boolean',
1842         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
1843         optional => 1,
1844     },
1845 };
1846
1847 sub parse_smbios1 {
1848     my ($data) = @_;
1849
1850     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
1851     warn $@ if $@;
1852     return $res;
1853 }
1854
1855 sub print_smbios1 {
1856     my ($smbios1) = @_;
1857     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1858 }
1859
1860 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1861
1862 sub parse_watchdog {
1863     my ($value) = @_;
1864
1865     return undef if !$value;
1866
1867     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
1868     warn $@ if $@;
1869     return $res;
1870 }
1871
1872 sub parse_guest_agent {
1873     my ($value) = @_;
1874
1875     return {} if !defined($value->{agent});
1876
1877     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
1878     warn $@ if $@;
1879
1880     # if the agent is disabled ignore the other potentially set properties
1881     return {} if !$res->{enabled};
1882     return $res;
1883 }
1884
1885 sub parse_vga {
1886     my ($value) = @_;
1887
1888     return {} if !$value;
1889     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
1890     warn $@ if $@;
1891     return $res;
1892 }
1893
1894 sub parse_rng {
1895     my ($value) = @_;
1896
1897     return undef if !$value;
1898
1899     my $res = eval { PVE::JSONSchema::parse_property_string($rng_fmt, $value) };
1900     warn $@ if $@;
1901     return $res;
1902 }
1903
1904 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1905 sub verify_usb_device {
1906     my ($value, $noerr) = @_;
1907
1908     return $value if parse_usb_device($value);
1909
1910     return undef if $noerr;
1911
1912     die "unable to parse usb device\n";
1913 }
1914
1915 # add JSON properties for create and set function
1916 sub json_config_properties {
1917     my $prop = shift;
1918
1919     foreach my $opt (keys %$confdesc) {
1920         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' ||
1921             $opt eq 'runningmachine' || $opt eq 'runningcpu';
1922         $prop->{$opt} = $confdesc->{$opt};
1923     }
1924
1925     return $prop;
1926 }
1927
1928 # return copy of $confdesc_cloudinit to generate documentation
1929 sub cloudinit_config_properties {
1930
1931     return dclone($confdesc_cloudinit);
1932 }
1933
1934 sub check_type {
1935     my ($key, $value) = @_;
1936
1937     die "unknown setting '$key'\n" if !$confdesc->{$key};
1938
1939     my $type = $confdesc->{$key}->{type};
1940
1941     if (!defined($value)) {
1942         die "got undefined value\n";
1943     }
1944
1945     if ($value =~ m/[\n\r]/) {
1946         die "property contains a line feed\n";
1947     }
1948
1949     if ($type eq 'boolean') {
1950         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1951         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1952         die "type check ('boolean') failed - got '$value'\n";
1953     } elsif ($type eq 'integer') {
1954         return int($1) if $value =~ m/^(\d+)$/;
1955         die "type check ('integer') failed - got '$value'\n";
1956     } elsif ($type eq 'number') {
1957         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1958         die "type check ('number') failed - got '$value'\n";
1959     } elsif ($type eq 'string') {
1960         if (my $fmt = $confdesc->{$key}->{format}) {
1961             PVE::JSONSchema::check_format($fmt, $value);
1962             return $value;
1963         }
1964         $value =~ s/^\"(.*)\"$/$1/;
1965         return $value;
1966     } else {
1967         die "internal error"
1968     }
1969 }
1970
1971 sub destroy_vm {
1972     my ($storecfg, $vmid, $skiplock, $replacement_conf) = @_;
1973
1974     my $conf = PVE::QemuConfig->load_config($vmid);
1975
1976     PVE::QemuConfig->check_lock($conf) if !$skiplock;
1977
1978     if ($conf->{template}) {
1979         # check if any base image is still used by a linked clone
1980         PVE::QemuConfig->foreach_volume($conf, sub {
1981                 my ($ds, $drive) = @_;
1982                 return if drive_is_cdrom($drive);
1983
1984                 my $volid = $drive->{file};
1985                 return if !$volid || $volid =~ m|^/|;
1986
1987                 die "base volume '$volid' is still in use by linked cloned\n"
1988                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
1989
1990         });
1991     }
1992
1993     # only remove disks owned by this VM
1994     PVE::QemuConfig->foreach_volume($conf, sub {
1995         my ($ds, $drive) = @_;
1996         return if drive_is_cdrom($drive, 1);
1997
1998         my $volid = $drive->{file};
1999         return if !$volid || $volid =~ m|^/|;
2000
2001         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2002         return if !$path || !$owner || ($owner != $vmid);
2003
2004         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2005         warn "Could not remove disk '$volid', check manually: $@" if $@;
2006     });
2007
2008     # also remove unused disk
2009     my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2010     PVE::Storage::foreach_volid($vmdisks, sub {
2011         my ($volid, $sid, $volname, $d) = @_;
2012         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2013         warn $@ if $@;
2014     });
2015
2016     if (defined $replacement_conf) {
2017         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2018     } else {
2019         PVE::QemuConfig->destroy_config($vmid);
2020     }
2021 }
2022
2023 sub parse_vm_config {
2024     my ($filename, $raw) = @_;
2025
2026     return undef if !defined($raw);
2027
2028     my $res = {
2029         digest => Digest::SHA::sha1_hex($raw),
2030         snapshots => {},
2031         pending => {},
2032     };
2033
2034     $filename =~ m|/qemu-server/(\d+)\.conf$|
2035         || die "got strange filename '$filename'";
2036
2037     my $vmid = $1;
2038
2039     my $conf = $res;
2040     my $descr;
2041     my $section = '';
2042
2043     my @lines = split(/\n/, $raw);
2044     foreach my $line (@lines) {
2045         next if $line =~ m/^\s*$/;
2046
2047         if ($line =~ m/^\[PENDING\]\s*$/i) {
2048             $section = 'pending';
2049             if (defined($descr)) {
2050                 $descr =~ s/\s+$//;
2051                 $conf->{description} = $descr;
2052             }
2053             $descr = undef;
2054             $conf = $res->{$section} = {};
2055             next;
2056
2057         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2058             $section = $1;
2059             if (defined($descr)) {
2060                 $descr =~ s/\s+$//;
2061                 $conf->{description} = $descr;
2062             }
2063             $descr = undef;
2064             $conf = $res->{snapshots}->{$section} = {};
2065             next;
2066         }
2067
2068         if ($line =~ m/^\#(.*)\s*$/) {
2069             $descr = '' if !defined($descr);
2070             $descr .= PVE::Tools::decode_text($1) . "\n";
2071             next;
2072         }
2073
2074         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2075             $descr = '' if !defined($descr);
2076             $descr .= PVE::Tools::decode_text($2);
2077         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2078             $conf->{snapstate} = $1;
2079         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2080             my $key = $1;
2081             my $value = $2;
2082             $conf->{$key} = $value;
2083         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2084             my $value = $1;
2085             if ($section eq 'pending') {
2086                 $conf->{delete} = $value; # we parse this later
2087             } else {
2088                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2089             }
2090         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2091             my $key = $1;
2092             my $value = $2;
2093             eval { $value = check_type($key, $value); };
2094             if ($@) {
2095                 warn "vm $vmid - unable to parse value of '$key' - $@";
2096             } else {
2097                 $key = 'ide2' if $key eq 'cdrom';
2098                 my $fmt = $confdesc->{$key}->{format};
2099                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2100                     my $v = parse_drive($key, $value);
2101                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2102                         $v->{file} = $volid;
2103                         $value = print_drive($v);
2104                     } else {
2105                         warn "vm $vmid - unable to parse value of '$key'\n";
2106                         next;
2107                     }
2108                 }
2109
2110                 $conf->{$key} = $value;
2111             }
2112         }
2113     }
2114
2115     if (defined($descr)) {
2116         $descr =~ s/\s+$//;
2117         $conf->{description} = $descr;
2118     }
2119     delete $res->{snapstate}; # just to be sure
2120
2121     return $res;
2122 }
2123
2124 sub write_vm_config {
2125     my ($filename, $conf) = @_;
2126
2127     delete $conf->{snapstate}; # just to be sure
2128
2129     if ($conf->{cdrom}) {
2130         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2131         $conf->{ide2} = $conf->{cdrom};
2132         delete $conf->{cdrom};
2133     }
2134
2135     # we do not use 'smp' any longer
2136     if ($conf->{sockets}) {
2137         delete $conf->{smp};
2138     } elsif ($conf->{smp}) {
2139         $conf->{sockets} = $conf->{smp};
2140         delete $conf->{cores};
2141         delete $conf->{smp};
2142     }
2143
2144     my $used_volids = {};
2145
2146     my $cleanup_config = sub {
2147         my ($cref, $pending, $snapname) = @_;
2148
2149         foreach my $key (keys %$cref) {
2150             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2151                 $key eq 'snapstate' || $key eq 'pending';
2152             my $value = $cref->{$key};
2153             if ($key eq 'delete') {
2154                 die "propertry 'delete' is only allowed in [PENDING]\n"
2155                     if !$pending;
2156                 # fixme: check syntax?
2157                 next;
2158             }
2159             eval { $value = check_type($key, $value); };
2160             die "unable to parse value of '$key' - $@" if $@;
2161
2162             $cref->{$key} = $value;
2163
2164             if (!$snapname && is_valid_drivename($key)) {
2165                 my $drive = parse_drive($key, $value);
2166                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2167             }
2168         }
2169     };
2170
2171     &$cleanup_config($conf);
2172
2173     &$cleanup_config($conf->{pending}, 1);
2174
2175     foreach my $snapname (keys %{$conf->{snapshots}}) {
2176         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2177         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2178     }
2179
2180     # remove 'unusedX' settings if we re-add a volume
2181     foreach my $key (keys %$conf) {
2182         my $value = $conf->{$key};
2183         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2184             delete $conf->{$key};
2185         }
2186     }
2187
2188     my $generate_raw_config = sub {
2189         my ($conf, $pending) = @_;
2190
2191         my $raw = '';
2192
2193         # add description as comment to top of file
2194         if (defined(my $descr = $conf->{description})) {
2195             if ($descr) {
2196                 foreach my $cl (split(/\n/, $descr)) {
2197                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2198                 }
2199             } else {
2200                 $raw .= "#\n" if $pending;
2201             }
2202         }
2203
2204         foreach my $key (sort keys %$conf) {
2205             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2206             $raw .= "$key: $conf->{$key}\n";
2207         }
2208         return $raw;
2209     };
2210
2211     my $raw = &$generate_raw_config($conf);
2212
2213     if (scalar(keys %{$conf->{pending}})){
2214         $raw .= "\n[PENDING]\n";
2215         $raw .= &$generate_raw_config($conf->{pending}, 1);
2216     }
2217
2218     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2219         $raw .= "\n[$snapname]\n";
2220         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2221     }
2222
2223     return $raw;
2224 }
2225
2226 sub load_defaults {
2227
2228     my $res = {};
2229
2230     # we use static defaults from our JSON schema configuration
2231     foreach my $key (keys %$confdesc) {
2232         if (defined(my $default = $confdesc->{$key}->{default})) {
2233             $res->{$key} = $default;
2234         }
2235     }
2236
2237     return $res;
2238 }
2239
2240 sub config_list {
2241     my $vmlist = PVE::Cluster::get_vmlist();
2242     my $res = {};
2243     return $res if !$vmlist || !$vmlist->{ids};
2244     my $ids = $vmlist->{ids};
2245     my $nodename = nodename();
2246
2247     foreach my $vmid (keys %$ids) {
2248         my $d = $ids->{$vmid};
2249         next if !$d->{node} || $d->{node} ne $nodename;
2250         next if !$d->{type} || $d->{type} ne 'qemu';
2251         $res->{$vmid}->{exists} = 1;
2252     }
2253     return $res;
2254 }
2255
2256 # test if VM uses local resources (to prevent migration)
2257 sub check_local_resources {
2258     my ($conf, $noerr) = @_;
2259
2260     my @loc_res = ();
2261
2262     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2263     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2264
2265     push @loc_res, "ivshmem" if $conf->{ivshmem};
2266
2267     foreach my $k (keys %$conf) {
2268         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2269         # sockets are safe: they will recreated be on the target side post-migrate
2270         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2271         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2272     }
2273
2274     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2275
2276     return \@loc_res;
2277 }
2278
2279 # check if used storages are available on all nodes (use by migrate)
2280 sub check_storage_availability {
2281     my ($storecfg, $conf, $node) = @_;
2282
2283     PVE::QemuConfig->foreach_volume($conf, sub {
2284         my ($ds, $drive) = @_;
2285
2286         my $volid = $drive->{file};
2287         return if !$volid;
2288
2289         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2290         return if !$sid;
2291
2292         # check if storage is available on both nodes
2293         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2294         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2295    });
2296 }
2297
2298 # list nodes where all VM images are available (used by has_feature API)
2299 sub shared_nodes {
2300     my ($conf, $storecfg) = @_;
2301
2302     my $nodelist = PVE::Cluster::get_nodelist();
2303     my $nodehash = { map { $_ => 1 } @$nodelist };
2304     my $nodename = nodename();
2305
2306     PVE::QemuConfig->foreach_volume($conf, sub {
2307         my ($ds, $drive) = @_;
2308
2309         my $volid = $drive->{file};
2310         return if !$volid;
2311
2312         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2313         if ($storeid) {
2314             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2315             if ($scfg->{disable}) {
2316                 $nodehash = {};
2317             } elsif (my $avail = $scfg->{nodes}) {
2318                 foreach my $node (keys %$nodehash) {
2319                     delete $nodehash->{$node} if !$avail->{$node};
2320                 }
2321             } elsif (!$scfg->{shared}) {
2322                 foreach my $node (keys %$nodehash) {
2323                     delete $nodehash->{$node} if $node ne $nodename
2324                 }
2325             }
2326         }
2327     });
2328
2329     return $nodehash
2330 }
2331
2332 sub check_local_storage_availability {
2333     my ($conf, $storecfg) = @_;
2334
2335     my $nodelist = PVE::Cluster::get_nodelist();
2336     my $nodehash = { map { $_ => {} } @$nodelist };
2337
2338     PVE::QemuConfig->foreach_volume($conf, sub {
2339         my ($ds, $drive) = @_;
2340
2341         my $volid = $drive->{file};
2342         return if !$volid;
2343
2344         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2345         if ($storeid) {
2346             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2347
2348             if ($scfg->{disable}) {
2349                 foreach my $node (keys %$nodehash) {
2350                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2351                 }
2352             } elsif (my $avail = $scfg->{nodes}) {
2353                 foreach my $node (keys %$nodehash) {
2354                     if (!$avail->{$node}) {
2355                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2356                     }
2357                 }
2358             }
2359         }
2360     });
2361
2362     foreach my $node (values %$nodehash) {
2363         if (my $unavail = $node->{unavailable_storages}) {
2364             $node->{unavailable_storages} = [ sort keys %$unavail ];
2365         }
2366     }
2367
2368     return $nodehash
2369 }
2370
2371 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2372 sub check_running {
2373     my ($vmid, $nocheck, $node) = @_;
2374
2375     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2376     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2377 }
2378
2379 sub vzlist {
2380
2381     my $vzlist = config_list();
2382
2383     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2384
2385     while (defined(my $de = $fd->read)) {
2386         next if $de !~ m/^(\d+)\.pid$/;
2387         my $vmid = $1;
2388         next if !defined($vzlist->{$vmid});
2389         if (my $pid = check_running($vmid)) {
2390             $vzlist->{$vmid}->{pid} = $pid;
2391         }
2392     }
2393
2394     return $vzlist;
2395 }
2396
2397 our $vmstatus_return_properties = {
2398     vmid => get_standard_option('pve-vmid'),
2399     status => {
2400         description => "Qemu process status.",
2401         type => 'string',
2402         enum => ['stopped', 'running'],
2403     },
2404     maxmem => {
2405         description => "Maximum memory in bytes.",
2406         type => 'integer',
2407         optional => 1,
2408         renderer => 'bytes',
2409     },
2410     maxdisk => {
2411         description => "Root disk size in bytes.",
2412         type => 'integer',
2413         optional => 1,
2414         renderer => 'bytes',
2415     },
2416     name => {
2417         description => "VM name.",
2418         type => 'string',
2419         optional => 1,
2420     },
2421     qmpstatus => {
2422         description => "Qemu QMP agent status.",
2423         type => 'string',
2424         optional => 1,
2425     },
2426     pid => {
2427         description => "PID of running qemu process.",
2428         type => 'integer',
2429         optional => 1,
2430     },
2431     uptime => {
2432         description => "Uptime.",
2433         type => 'integer',
2434         optional => 1,
2435         renderer => 'duration',
2436     },
2437     cpus => {
2438         description => "Maximum usable CPUs.",
2439         type => 'number',
2440         optional => 1,
2441     },
2442     lock => {
2443         description => "The current config lock, if any.",
2444         type => 'string',
2445         optional => 1,
2446     },
2447     tags => {
2448         description  => "The current configured tags, if any",
2449         type => 'string',
2450         optional => 1,
2451     },
2452 };
2453
2454 my $last_proc_pid_stat;
2455
2456 # get VM status information
2457 # This must be fast and should not block ($full == false)
2458 # We only query KVM using QMP if $full == true (this can be slow)
2459 sub vmstatus {
2460     my ($opt_vmid, $full) = @_;
2461
2462     my $res = {};
2463
2464     my $storecfg = PVE::Storage::config();
2465
2466     my $list = vzlist();
2467     my $defaults = load_defaults();
2468
2469     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2470
2471     my $cpucount = $cpuinfo->{cpus} || 1;
2472
2473     foreach my $vmid (keys %$list) {
2474         next if $opt_vmid && ($vmid ne $opt_vmid);
2475
2476         my $conf = PVE::QemuConfig->load_config($vmid);
2477
2478         my $d = { vmid => $vmid };
2479         $d->{pid} = $list->{$vmid}->{pid};
2480
2481         # fixme: better status?
2482         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2483
2484         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2485         if (defined($size)) {
2486             $d->{disk} = 0; # no info available
2487             $d->{maxdisk} = $size;
2488         } else {
2489             $d->{disk} = 0;
2490             $d->{maxdisk} = 0;
2491         }
2492
2493         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2494             * ($conf->{cores} || $defaults->{cores});
2495         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2496         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2497
2498         $d->{name} = $conf->{name} || "VM $vmid";
2499         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2500             : $defaults->{memory}*(1024*1024);
2501
2502         if ($conf->{balloon}) {
2503             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2504             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2505                 : $defaults->{shares};
2506         }
2507
2508         $d->{uptime} = 0;
2509         $d->{cpu} = 0;
2510         $d->{mem} = 0;
2511
2512         $d->{netout} = 0;
2513         $d->{netin} = 0;
2514
2515         $d->{diskread} = 0;
2516         $d->{diskwrite} = 0;
2517
2518         $d->{template} = PVE::QemuConfig->is_template($conf);
2519
2520         $d->{serial} = 1 if conf_has_serial($conf);
2521         $d->{lock} = $conf->{lock} if $conf->{lock};
2522         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2523
2524         $res->{$vmid} = $d;
2525     }
2526
2527     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2528     foreach my $dev (keys %$netdev) {
2529         next if $dev !~ m/^tap([1-9]\d*)i/;
2530         my $vmid = $1;
2531         my $d = $res->{$vmid};
2532         next if !$d;
2533
2534         $d->{netout} += $netdev->{$dev}->{receive};
2535         $d->{netin} += $netdev->{$dev}->{transmit};
2536
2537         if ($full) {
2538             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2539             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2540         }
2541
2542     }
2543
2544     my $ctime = gettimeofday;
2545
2546     foreach my $vmid (keys %$list) {
2547
2548         my $d = $res->{$vmid};
2549         my $pid = $d->{pid};
2550         next if !$pid;
2551
2552         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2553         next if !$pstat; # not running
2554
2555         my $used = $pstat->{utime} + $pstat->{stime};
2556
2557         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2558
2559         if ($pstat->{vsize}) {
2560             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2561         }
2562
2563         my $old = $last_proc_pid_stat->{$pid};
2564         if (!$old) {
2565             $last_proc_pid_stat->{$pid} = {
2566                 time => $ctime,
2567                 used => $used,
2568                 cpu => 0,
2569             };
2570             next;
2571         }
2572
2573         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2574
2575         if ($dtime > 1000) {
2576             my $dutime = $used -  $old->{used};
2577
2578             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2579             $last_proc_pid_stat->{$pid} = {
2580                 time => $ctime,
2581                 used => $used,
2582                 cpu => $d->{cpu},
2583             };
2584         } else {
2585             $d->{cpu} = $old->{cpu};
2586         }
2587     }
2588
2589     return $res if !$full;
2590
2591     my $qmpclient = PVE::QMPClient->new();
2592
2593     my $ballooncb = sub {
2594         my ($vmid, $resp) = @_;
2595
2596         my $info = $resp->{'return'};
2597         return if !$info->{max_mem};
2598
2599         my $d = $res->{$vmid};
2600
2601         # use memory assigned to VM
2602         $d->{maxmem} = $info->{max_mem};
2603         $d->{balloon} = $info->{actual};
2604
2605         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2606             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2607             $d->{freemem} = $info->{free_mem};
2608         }
2609
2610         $d->{ballooninfo} = $info;
2611     };
2612
2613     my $blockstatscb = sub {
2614         my ($vmid, $resp) = @_;
2615         my $data = $resp->{'return'} || [];
2616         my $totalrdbytes = 0;
2617         my $totalwrbytes = 0;
2618
2619         for my $blockstat (@$data) {
2620             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2621             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2622
2623             $blockstat->{device} =~ s/drive-//;
2624             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2625         }
2626         $res->{$vmid}->{diskread} = $totalrdbytes;
2627         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2628     };
2629
2630     my $statuscb = sub {
2631         my ($vmid, $resp) = @_;
2632
2633         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2634         # this fails if ballon driver is not loaded, so this must be
2635         # the last commnand (following command are aborted if this fails).
2636         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2637
2638         my $status = 'unknown';
2639         if (!defined($status = $resp->{'return'}->{status})) {
2640             warn "unable to get VM status\n";
2641             return;
2642         }
2643
2644         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2645     };
2646
2647     foreach my $vmid (keys %$list) {
2648         next if $opt_vmid && ($vmid ne $opt_vmid);
2649         next if !$res->{$vmid}->{pid}; # not running
2650         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2651     }
2652
2653     $qmpclient->queue_execute(undef, 2);
2654
2655     foreach my $vmid (keys %$list) {
2656         next if $opt_vmid && ($vmid ne $opt_vmid);
2657         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2658     }
2659
2660     return $res;
2661 }
2662
2663 sub conf_has_serial {
2664     my ($conf) = @_;
2665
2666     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2667         if ($conf->{"serial$i"}) {
2668             return 1;
2669         }
2670     }
2671
2672     return 0;
2673 }
2674
2675 sub conf_has_audio {
2676     my ($conf, $id) = @_;
2677
2678     $id //= 0;
2679     my $audio = $conf->{"audio$id"};
2680     return undef if !defined($audio);
2681
2682     my $audioproperties = PVE::JSONSchema::parse_property_string($audio_fmt, $audio);
2683     my $audiodriver = $audioproperties->{driver} // 'spice';
2684
2685     return {
2686         dev => $audioproperties->{device},
2687         dev_id => "audiodev$id",
2688         backend => $audiodriver,
2689         backend_id => "$audiodriver-backend${id}",
2690     };
2691 }
2692
2693 sub audio_devs {
2694     my ($audio, $audiopciaddr, $machine_version) = @_;
2695
2696     my $devs = [];
2697
2698     my $id = $audio->{dev_id};
2699     my $audiodev = "";
2700     if (min_version($machine_version, 4, 2)) {
2701         $audiodev = ",audiodev=$audio->{backend_id}";
2702     }
2703
2704     if ($audio->{dev} eq 'AC97') {
2705         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
2706     } elsif ($audio->{dev} =~ /intel\-hda$/) {
2707         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
2708         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
2709         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
2710     } else {
2711         die "unkown audio device '$audio->{dev}', implement me!";
2712     }
2713
2714     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
2715
2716     return $devs;
2717 }
2718
2719 sub vga_conf_has_spice {
2720     my ($vga) = @_;
2721
2722     my $vgaconf = parse_vga($vga);
2723     my $vgatype = $vgaconf->{type};
2724     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
2725
2726     return $1 || 1;
2727 }
2728
2729 sub is_native($) {
2730     my ($arch) = @_;
2731     return get_host_arch() eq $arch;
2732 }
2733
2734 sub get_vm_arch {
2735     my ($conf) = @_;
2736     return $conf->{arch} // get_host_arch();
2737 }
2738
2739 my $default_machines = {
2740     x86_64 => 'pc',
2741     aarch64 => 'virt',
2742 };
2743
2744 sub get_vm_machine {
2745     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
2746
2747     my $machine = $forcemachine || $conf->{machine};
2748
2749     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
2750         $arch //= 'x86_64';
2751         $machine ||= $default_machines->{$arch};
2752         if ($add_pve_version) {
2753             $kvmversion //= kvm_user_version();
2754             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
2755             $machine .= "+pve$pvever";
2756         }
2757     }
2758
2759     if ($add_pve_version && $machine !~ m/\+pve\d+$/) {
2760         # for version-pinned machines that do not include a pve-version (e.g.
2761         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
2762         $machine .= '+pve0';
2763     }
2764
2765     return $machine;
2766 }
2767
2768 sub get_ovmf_files($) {
2769     my ($arch) = @_;
2770
2771     my $ovmf = $OVMF->{$arch}
2772         or die "no OVMF images known for architecture '$arch'\n";
2773
2774     return @$ovmf;
2775 }
2776
2777 my $Arch2Qemu = {
2778     aarch64 => '/usr/bin/qemu-system-aarch64',
2779     x86_64 => '/usr/bin/qemu-system-x86_64',
2780 };
2781 sub get_command_for_arch($) {
2782     my ($arch) = @_;
2783     return '/usr/bin/kvm' if is_native($arch);
2784
2785     my $cmd = $Arch2Qemu->{$arch}
2786         or die "don't know how to emulate architecture '$arch'\n";
2787     return $cmd;
2788 }
2789
2790 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
2791 # to use in a QEMU command line (-cpu element), first array_intersect the result
2792 # of query_supported_ with query_understood_. This is necessary because:
2793 #
2794 # a) query_understood_ returns flags the host cannot use and
2795 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
2796 #    flags, but CPU settings - with most of them being flags. Those settings
2797 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
2798 #
2799 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
2800 # expensive. If you need the value returned from this, you can get it much
2801 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
2802 # $accel being 'kvm' or 'tcg'.
2803 #
2804 # pvestatd calls this function on startup and whenever the QEMU/KVM version
2805 # changes, automatically populating pmxcfs.
2806 #
2807 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
2808 # since kvm and tcg machines support different flags
2809 #
2810 sub query_supported_cpu_flags {
2811     my ($arch) = @_;
2812
2813     $arch //= get_host_arch();
2814     my $default_machine = $default_machines->{$arch};
2815
2816     my $flags = {};
2817
2818     # FIXME: Once this is merged, the code below should work for ARM as well:
2819     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
2820     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
2821         $arch eq "aarch64";
2822
2823     my $kvm_supported = defined(kvm_version());
2824     my $qemu_cmd = get_command_for_arch($arch);
2825     my $fakevmid = -1;
2826     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
2827
2828     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
2829     my $query_supported_run_qemu = sub {
2830         my ($kvm) = @_;
2831
2832         my $flags = {};
2833         my $cmd = [
2834             $qemu_cmd,
2835             '-machine', $default_machine,
2836             '-display', 'none',
2837             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server,nowait",
2838             '-mon', 'chardev=qmp,mode=control',
2839             '-pidfile', $pidfile,
2840             '-S', '-daemonize'
2841         ];
2842
2843         if (!$kvm) {
2844             push @$cmd, '-accel', 'tcg';
2845         }
2846
2847         my $rc = run_command($cmd, noerr => 1, quiet => 0);
2848         die "QEMU flag querying VM exited with code " . $rc if $rc;
2849
2850         eval {
2851             my $cmd_result = mon_cmd(
2852                 $fakevmid,
2853                 'query-cpu-model-expansion',
2854                 type => 'full',
2855                 model => { name => 'host' }
2856             );
2857
2858             my $props = $cmd_result->{model}->{props};
2859             foreach my $prop (keys %$props) {
2860                 next if $props->{$prop} ne '1';
2861                 # QEMU returns some flags multiple times, with '_', '.' or '-'
2862                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
2863                 # We only keep those with underscores, to match /proc/cpuinfo
2864                 $prop =~ s/\.|-/_/g;
2865                 $flags->{$prop} = 1;
2866             }
2867         };
2868         my $err = $@;
2869
2870         # force stop with 10 sec timeout and 'nocheck'
2871         # always stop, even if QMP failed
2872         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
2873
2874         die $err if $err;
2875
2876         return [ sort keys %$flags ];
2877     };
2878
2879     # We need to query QEMU twice, since KVM and TCG have different supported flags
2880     PVE::QemuConfig->lock_config($fakevmid, sub {
2881         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
2882         warn "warning: failed querying supported tcg flags: $@\n" if $@;
2883
2884         if ($kvm_supported) {
2885             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
2886             warn "warning: failed querying supported kvm flags: $@\n" if $@;
2887         }
2888     });
2889
2890     return $flags;
2891 }
2892
2893 # Understood CPU flags are written to a file at 'pve-qemu' compile time
2894 my $understood_cpu_flag_dir = "/usr/share/kvm";
2895 sub query_understood_cpu_flags {
2896     my $arch = get_host_arch();
2897     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
2898
2899     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
2900         if ! -e $filepath;
2901
2902     my $raw = file_get_contents($filepath);
2903     $raw =~ s/^\s+|\s+$//g;
2904     my @flags = split(/\s+/, $raw);
2905
2906     return \@flags;
2907 }
2908
2909 sub config_to_command {
2910     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu) = @_;
2911
2912     my $cmd = [];
2913     my $globalFlags = [];
2914     my $machineFlags = [];
2915     my $rtcFlags = [];
2916     my $devices = [];
2917     my $pciaddr = '';
2918     my $bridges = {};
2919     my $ostype = $conf->{ostype};
2920     my $winversion = windows_version($ostype);
2921     my $kvm = $conf->{kvm};
2922     my $nodename = nodename();
2923
2924     my $arch = get_vm_arch($conf);
2925     my $kvm_binary = get_command_for_arch($arch);
2926     my $kvmver = kvm_user_version($kvm_binary);
2927
2928     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
2929         $kvmver //= "undefined";
2930         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
2931     }
2932
2933     my $add_pve_version = min_version($kvmver, 4, 1);
2934
2935     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
2936     my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type, $kvmver);
2937     $kvm //= 1 if is_native($arch);
2938
2939     $machine_version =~ m/(\d+)\.(\d+)/;
2940     my ($machine_major, $machine_minor) = ($1, $2);
2941
2942     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
2943         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
2944     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
2945         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type', please upgrade node '$nodename'\n"
2946     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
2947         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
2948         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is pve$max_pve_version)"
2949             ." is too old to run machine type '$machine_type', please upgrade node '$nodename'\n";
2950     }
2951
2952     # if a specific +pve version is required for a feature, use $version_guard
2953     # instead of min_version to allow machines to be run with the minimum
2954     # required version
2955     my $required_pve_version = 0;
2956     my $version_guard = sub {
2957         my ($major, $minor, $pve) = @_;
2958         return 0 if !min_version($machine_version, $major, $minor, $pve);
2959         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
2960         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
2961         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
2962         return 1;
2963     };
2964
2965     if ($kvm) {
2966         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
2967             if !defined kvm_version();
2968     }
2969
2970     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2971     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2972     my $use_old_bios_files = undef;
2973     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2974
2975     my $cpuunits = defined($conf->{cpuunits}) ?
2976             $conf->{cpuunits} : $defaults->{cpuunits};
2977
2978     push @$cmd, $kvm_binary;
2979
2980     push @$cmd, '-id', $vmid;
2981
2982     my $vmname = $conf->{name} || "vm$vmid";
2983
2984     push @$cmd, '-name', $vmname;
2985
2986     my $use_virtio = 0;
2987
2988     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
2989     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2990     push @$cmd, '-mon', "chardev=qmp,mode=control";
2991
2992     if (min_version($machine_version, 2, 12)) {
2993         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
2994         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
2995     }
2996
2997     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
2998
2999     push @$cmd, '-daemonize';
3000
3001     if ($conf->{smbios1}) {
3002         my $smbios_conf = parse_smbios1($conf->{smbios1});
3003         if ($smbios_conf->{base64}) {
3004             # Do not pass base64 flag to qemu
3005             delete $smbios_conf->{base64};
3006             my $smbios_string = "";
3007             foreach my $key (keys %$smbios_conf) {
3008                 my $value;
3009                 if ($key eq "uuid") {
3010                     $value = $smbios_conf->{uuid}
3011                 } else {
3012                     $value = decode_base64($smbios_conf->{$key});
3013                 }
3014                 # qemu accepts any binary data, only commas need escaping by double comma
3015                 $value =~ s/,/,,/g;
3016                 $smbios_string .= "," . $key . "=" . $value if $value;
3017             }
3018             push @$cmd, '-smbios', "type=1" . $smbios_string;
3019         } else {
3020             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3021         }
3022     }
3023
3024     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3025         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3026         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3027
3028         my ($path, $format);
3029         if (my $efidisk = $conf->{efidisk0}) {
3030             my $d = parse_drive('efidisk0', $efidisk);
3031             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3032             $format = $d->{format};
3033             if ($storeid) {
3034                 $path = PVE::Storage::path($storecfg, $d->{file});
3035                 if (!defined($format)) {
3036                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3037                     $format = qemu_img_format($scfg, $volname);
3038                 }
3039             } else {
3040                 $path = $d->{file};
3041                 die "efidisk format must be specified\n"
3042                     if !defined($format);
3043             }
3044         } else {
3045             warn "no efidisk configured! Using temporary efivars disk.\n";
3046             $path = "/tmp/$vmid-ovmf.fd";
3047             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3048             $format = 'raw';
3049         }
3050
3051         my $size_str = "";
3052
3053         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3054             $size_str = ",size=" . (-s $ovmf_vars);
3055         }
3056
3057         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3058         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0$size_str,file=$path";
3059     }
3060
3061     # load q35 config
3062     if ($q35) {
3063         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3064         if (min_version($machine_version, 4, 0)) {
3065             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3066         } else {
3067             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3068         }
3069     }
3070
3071     if ($conf->{vmgenid}) {
3072         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3073     }
3074
3075     # add usb controllers
3076     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3077     push @$devices, @usbcontrollers if @usbcontrollers;
3078     my $vga = parse_vga($conf->{vga});
3079
3080     my $qxlnum = vga_conf_has_spice($conf->{vga});
3081     $vga->{type} = 'qxl' if $qxlnum;
3082
3083     if (!$vga->{type}) {
3084         if ($arch eq 'aarch64') {
3085             $vga->{type} = 'virtio';
3086         } elsif (min_version($machine_version, 2, 9)) {
3087             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3088         } else {
3089             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3090         }
3091     }
3092
3093     # enable absolute mouse coordinates (needed by vnc)
3094     my $tablet;
3095     if (defined($conf->{tablet})) {
3096         $tablet = $conf->{tablet};
3097     } else {
3098         $tablet = $defaults->{tablet};
3099         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3100         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3101     }
3102
3103     if ($tablet) {
3104         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3105         my $kbd = print_keyboarddevice_full($conf, $arch);
3106         push @$devices, '-device', $kbd if defined($kbd);
3107     }
3108
3109     # host pci device passthrough
3110     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3111         $vmid, $conf, $devices, $winversion, $q35, $bridges, $arch, $machine_type);
3112
3113     # usb devices
3114     my $usb_dev_features = {};
3115     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3116
3117     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features);
3118     push @$devices, @usbdevices if @usbdevices;
3119     # serial devices
3120     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3121         if (my $path = $conf->{"serial$i"}) {
3122             if ($path eq 'socket') {
3123                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3124                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3125                 # On aarch64, serial0 is the UART device. Qemu only allows
3126                 # connecting UART devices via the '-serial' command line, as
3127                 # the device has a fixed slot on the hardware...
3128                 if ($arch eq 'aarch64' && $i == 0) {
3129                     push @$devices, '-serial', "chardev:serial$i";
3130                 } else {
3131                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3132                 }
3133             } else {
3134                 die "no such serial device\n" if ! -c $path;
3135                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3136                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3137             }
3138         }
3139     }
3140
3141     # parallel devices
3142     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3143         if (my $path = $conf->{"parallel$i"}) {
3144             die "no such parallel device\n" if ! -c $path;
3145             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3146             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3147             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3148         }
3149     }
3150
3151     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3152         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3153         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3154         push @$devices, @$audio_devs;
3155     }
3156
3157     my $sockets = 1;
3158     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3159     $sockets = $conf->{sockets} if  $conf->{sockets};
3160
3161     my $cores = $conf->{cores} || 1;
3162
3163     my $maxcpus = $sockets * $cores;
3164
3165     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3166
3167     my $allowed_vcpus = $cpuinfo->{cpus};
3168
3169     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3170         if ($allowed_vcpus < $maxcpus);
3171
3172     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3173
3174         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3175         for (my $i = 2; $i <= $vcpus; $i++)  {
3176             my $cpustr = print_cpu_device($conf,$i);
3177             push @$cmd, '-device', $cpustr;
3178         }
3179
3180     } else {
3181
3182         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3183     }
3184     push @$cmd, '-nodefaults';
3185
3186     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3187
3188     my $bootindex_hash = {};
3189     my $i = 1;
3190     foreach my $o (split(//, $bootorder)) {
3191         $bootindex_hash->{$o} = $i*100;
3192         $i++;
3193     }
3194
3195     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3196
3197     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3198
3199     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3200
3201     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3202         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3203         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3204         push @$cmd,  '-vnc', "unix:$socket,password";
3205     } else {
3206         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3207         push @$cmd, '-nographic';
3208     }
3209
3210     # time drift fix
3211     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3212     my $useLocaltime = $conf->{localtime};
3213
3214     if ($winversion >= 5) { # windows
3215         $useLocaltime = 1 if !defined($conf->{localtime});
3216
3217         # use time drift fix when acpi is enabled
3218         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3219             $tdf = 1 if !defined($conf->{tdf});
3220         }
3221     }
3222
3223     if ($winversion >= 6) {
3224         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3225         push @$cmd, '-no-hpet';
3226     }
3227
3228     push @$rtcFlags, 'driftfix=slew' if $tdf;
3229
3230     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3231         push @$rtcFlags, "base=$conf->{startdate}";
3232     } elsif ($useLocaltime) {
3233         push @$rtcFlags, 'base=localtime';
3234     }
3235
3236     if ($forcecpu) {
3237         push @$cmd, '-cpu', $forcecpu;
3238     } else {
3239         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3240     }
3241
3242     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3243
3244     push @$cmd, '-S' if $conf->{freeze};
3245
3246     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3247
3248     my $guest_agent = parse_guest_agent($conf);
3249
3250     if ($guest_agent->{enabled}) {
3251         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3252         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3253
3254         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3255             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3256             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3257             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3258         } elsif ($guest_agent->{type} eq 'isa') {
3259             push @$devices, '-device', "isa-serial,chardev=qga0";
3260         }
3261     }
3262
3263     my $rng = parse_rng($conf->{rng0}) if $conf->{rng0};
3264     if ($rng && &$version_guard(4, 1, 2)) {
3265         check_rng_source($rng->{source});
3266
3267         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3268         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3269         my $limiter_str = "";
3270         if ($max_bytes) {
3271             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3272         }
3273
3274         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3275         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3276         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3277     }
3278
3279     my $spice_port;
3280
3281     if ($qxlnum) {
3282         if ($qxlnum > 1) {
3283             if ($winversion){
3284                 for (my $i = 1; $i < $qxlnum; $i++){
3285                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3286                 }
3287             } else {
3288                 # assume other OS works like Linux
3289                 my ($ram, $vram) = ("134217728", "67108864");
3290                 if ($vga->{memory}) {
3291                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3292                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3293                 }
3294                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3295                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3296             }
3297         }
3298
3299         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3300
3301         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3302         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3303         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3304
3305         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3306         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3307         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3308
3309         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3310         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3311
3312         my $spice_enhancement = PVE::JSONSchema::parse_property_string($spice_enhancements_fmt, $conf->{spice_enhancements} // '');
3313         if ($spice_enhancement->{foldersharing}) {
3314             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3315             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3316         }
3317
3318         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3319         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming};
3320         push @$devices, '-spice', "$spice_opts";
3321     }
3322
3323     # enable balloon by default, unless explicitly disabled
3324     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3325         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3326         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3327     }
3328
3329     if ($conf->{watchdog}) {
3330         my $wdopts = parse_watchdog($conf->{watchdog});
3331         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3332         my $watchdog = $wdopts->{model} || 'i6300esb';
3333         push @$devices, '-device', "$watchdog$pciaddr";
3334         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3335     }
3336
3337     my $vollist = [];
3338     my $scsicontroller = {};
3339     my $ahcicontroller = {};
3340     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3341
3342     # Add iscsi initiator name if available
3343     if (my $initiator = get_initiator_name()) {
3344         push @$devices, '-iscsi', "initiator-name=$initiator";
3345     }
3346
3347     PVE::QemuConfig->foreach_volume($conf, sub {
3348         my ($ds, $drive) = @_;
3349
3350         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3351             push @$vollist, $drive->{file};
3352         }
3353
3354         # ignore efidisk here, already added in bios/fw handling code above
3355         return if $drive->{interface} eq 'efidisk';
3356
3357         $use_virtio = 1 if $ds =~ m/^virtio/;
3358
3359         if (drive_is_cdrom ($drive)) {
3360             if ($bootindex_hash->{d}) {
3361                 $drive->{bootindex} = $bootindex_hash->{d};
3362                 $bootindex_hash->{d} += 1;
3363             }
3364         } else {
3365             if ($bootindex_hash->{c}) {
3366                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3367                 $bootindex_hash->{c} += 1;
3368             }
3369         }
3370
3371         if ($drive->{interface} eq 'virtio'){
3372            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3373         }
3374
3375         if ($drive->{interface} eq 'scsi') {
3376
3377             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3378
3379             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3380                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3381
3382             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3383             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3384
3385             my $iothread = '';
3386             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3387                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3388                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3389             } elsif ($drive->{iothread}) {
3390                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3391             }
3392
3393             my $queues = '';
3394             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3395                 $queues = ",num_queues=$drive->{queues}";
3396             }
3397
3398             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3399             $scsicontroller->{$controller}=1;
3400         }
3401
3402         if ($drive->{interface} eq 'sata') {
3403             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3404             $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3405             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3406             $ahcicontroller->{$controller}=1;
3407         }
3408
3409         my $drive_cmd = print_drive_commandline_full($storecfg, $vmid, $drive);
3410         push @$devices, '-drive',$drive_cmd;
3411         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3412     });
3413
3414     for (my $i = 0; $i < $MAX_NETS; $i++) {
3415          next if !$conf->{"net$i"};
3416          my $d = parse_net($conf->{"net$i"});
3417          next if !$d;
3418
3419          $use_virtio = 1 if $d->{model} eq 'virtio';
3420
3421          if ($bootindex_hash->{n}) {
3422             $d->{bootindex} = $bootindex_hash->{n};
3423             $bootindex_hash->{n} += 1;
3424          }
3425
3426          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
3427          push @$devices, '-netdev', $netdevfull;
3428
3429          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
3430          push @$devices, '-device', $netdevicefull;
3431     }
3432
3433     if ($conf->{ivshmem}) {
3434         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3435
3436         my $bus;
3437         if ($q35) {
3438             $bus = print_pcie_addr("ivshmem");
3439         } else {
3440             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3441         }
3442
3443         my $ivshmem_name = $ivshmem->{name} // $vmid;
3444         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3445
3446         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3447         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
3448     }
3449
3450     # pci.4 is nested in pci.1
3451     $bridges->{1} = 1 if $bridges->{4};
3452
3453     if (!$q35) {
3454         # add pci bridges
3455         if (min_version($machine_version, 2, 3)) {
3456            $bridges->{1} = 1;
3457            $bridges->{2} = 1;
3458         }
3459
3460         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3461
3462     }
3463
3464     for my $k (sort {$b cmp $a} keys %$bridges) {
3465         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3466
3467         my $k_name = $k;
3468         if ($k == 2 && $legacy_igd) {
3469             $k_name = "$k-igd";
3470         }
3471         $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
3472
3473         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3474         if ($q35) {
3475             # add after -readconfig pve-q35.cfg
3476             splice @$devices, 2, 0, '-device', $devstr;
3477         } else {
3478             unshift @$devices, '-device', $devstr if $k > 0;
3479         }
3480     }
3481
3482     if (!$kvm) {
3483         push @$machineFlags, 'accel=tcg';
3484     }
3485
3486     my $machine_type_min = $machine_type;
3487     if ($add_pve_version) {
3488         $machine_type_min =~ s/\+pve\d+$//;
3489         $machine_type_min .= "+pve$required_pve_version";
3490     }
3491     push @$machineFlags, "type=${machine_type_min}";
3492
3493     push @$cmd, @$devices;
3494     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
3495     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
3496     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
3497
3498     if (my $vmstate = $conf->{vmstate}) {
3499         my $statepath = PVE::Storage::path($storecfg, $vmstate);
3500         push @$vollist, $vmstate;
3501         push @$cmd, '-loadstate', $statepath;
3502         print "activating and using '$vmstate' as vmstate\n";
3503     }
3504
3505     # add custom args
3506     if ($conf->{args}) {
3507         my $aa = PVE::Tools::split_args($conf->{args});
3508         push @$cmd, @$aa;
3509     }
3510
3511     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3512 }
3513
3514 sub check_rng_source {
3515     my ($source) = @_;
3516
3517     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3518     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
3519         if ! -e $source;
3520
3521     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
3522     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
3523         # Needs to abort, otherwise QEMU crashes on first rng access.
3524         # Note that rng_current cannot be changed to 'none' manually, so
3525         # once the VM is past this point, it is no longer an issue.
3526         die "Cannot start VM with passed-through RNG device: '/dev/hwrng'"
3527             . " exists, but '$rng_current' is set to 'none'. Ensure that"
3528             . " a compatible hardware-RNG is attached to the host.\n";
3529     }
3530 }
3531
3532 sub spice_port {
3533     my ($vmid) = @_;
3534
3535     my $res = mon_cmd($vmid, 'query-spice');
3536
3537     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3538 }
3539
3540 sub vm_devices_list {
3541     my ($vmid) = @_;
3542
3543     my $res = mon_cmd($vmid, 'query-pci');
3544     my $devices_to_check = [];
3545     my $devices = {};
3546     foreach my $pcibus (@$res) {
3547         push @$devices_to_check, @{$pcibus->{devices}},
3548     }
3549
3550     while (@$devices_to_check) {
3551         my $to_check = [];
3552         for my $d (@$devices_to_check) {
3553             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3554             next if !$d->{'pci_bridge'};
3555
3556             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3557             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3558         }
3559         $devices_to_check = $to_check;
3560     }
3561
3562     my $resblock = mon_cmd($vmid, 'query-block');
3563     foreach my $block (@$resblock) {
3564         if($block->{device} =~ m/^drive-(\S+)/){
3565                 $devices->{$1} = 1;
3566         }
3567     }
3568
3569     my $resmice = mon_cmd($vmid, 'query-mice');
3570     foreach my $mice (@$resmice) {
3571         if ($mice->{name} eq 'QEMU HID Tablet') {
3572             $devices->{tablet} = 1;
3573             last;
3574         }
3575     }
3576
3577     # for usb devices there is no query-usb
3578     # but we can iterate over the entries in
3579     # qom-list path=/machine/peripheral
3580     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3581     foreach my $per (@$resperipheral) {
3582         if ($per->{name} =~ m/^usb\d+$/) {
3583             $devices->{$per->{name}} = 1;
3584         }
3585     }
3586
3587     return $devices;
3588 }
3589
3590 sub vm_deviceplug {
3591     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3592
3593     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3594
3595     my $devices_list = vm_devices_list($vmid);
3596     return 1 if defined($devices_list->{$deviceid});
3597
3598     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
3599
3600     if ($deviceid eq 'tablet') {
3601
3602         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
3603
3604     } elsif ($deviceid eq 'keyboard') {
3605
3606         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
3607
3608     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3609
3610         die "usb hotplug currently not reliable\n";
3611         # since we can't reliably hot unplug all added usb devices
3612         # and usb passthrough disables live migration
3613         # we disable usb hotplugging for now
3614         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3615
3616     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3617
3618         qemu_iothread_add($vmid, $deviceid, $device);
3619
3620         qemu_driveadd($storecfg, $vmid, $device);
3621         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
3622
3623         qemu_deviceadd($vmid, $devicefull);
3624         eval { qemu_deviceaddverify($vmid, $deviceid); };
3625         if (my $err = $@) {
3626             eval { qemu_drivedel($vmid, $deviceid); };
3627             warn $@ if $@;
3628             die $err;
3629         }
3630
3631     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3632
3633
3634         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3635         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3636         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3637
3638         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3639
3640         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3641             qemu_iothread_add($vmid, $deviceid, $device);
3642             $devicefull .= ",iothread=iothread-$deviceid";
3643         }
3644
3645         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3646             $devicefull .= ",num_queues=$device->{queues}";
3647         }
3648
3649         qemu_deviceadd($vmid, $devicefull);
3650         qemu_deviceaddverify($vmid, $deviceid);
3651
3652     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3653
3654         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
3655         qemu_driveadd($storecfg, $vmid, $device);
3656
3657         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
3658         eval { qemu_deviceadd($vmid, $devicefull); };
3659         if (my $err = $@) {
3660             eval { qemu_drivedel($vmid, $deviceid); };
3661             warn $@ if $@;
3662             die $err;
3663         }
3664
3665     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3666
3667         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
3668
3669         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
3670         my $use_old_bios_files = undef;
3671         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3672
3673         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
3674         qemu_deviceadd($vmid, $netdevicefull);
3675         eval {
3676             qemu_deviceaddverify($vmid, $deviceid);
3677             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
3678         };
3679         if (my $err = $@) {
3680             eval { qemu_netdevdel($vmid, $deviceid); };
3681             warn $@ if $@;
3682             die $err;
3683         }
3684
3685     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3686
3687         my $bridgeid = $2;
3688         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3689         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3690
3691         qemu_deviceadd($vmid, $devicefull);
3692         qemu_deviceaddverify($vmid, $deviceid);
3693
3694     } else {
3695         die "can't hotplug device '$deviceid'\n";
3696     }
3697
3698     return 1;
3699 }
3700
3701 # fixme: this should raise exceptions on error!
3702 sub vm_deviceunplug {
3703     my ($vmid, $conf, $deviceid) = @_;
3704
3705     my $devices_list = vm_devices_list($vmid);
3706     return 1 if !defined($devices_list->{$deviceid});
3707
3708     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3709
3710     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
3711
3712         qemu_devicedel($vmid, $deviceid);
3713
3714     } elsif ($deviceid =~ m/^usb\d+$/) {
3715
3716         die "usb hotplug currently not reliable\n";
3717         # when unplugging usb devices this way,
3718         # there may be remaining usb controllers/hubs
3719         # so we disable it for now
3720         qemu_devicedel($vmid, $deviceid);
3721         qemu_devicedelverify($vmid, $deviceid);
3722
3723     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3724
3725         qemu_devicedel($vmid, $deviceid);
3726         qemu_devicedelverify($vmid, $deviceid);
3727         qemu_drivedel($vmid, $deviceid);
3728         qemu_iothread_del($conf, $vmid, $deviceid);
3729
3730     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3731
3732         qemu_devicedel($vmid, $deviceid);
3733         qemu_devicedelverify($vmid, $deviceid);
3734         qemu_iothread_del($conf, $vmid, $deviceid);
3735
3736     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3737
3738         qemu_devicedel($vmid, $deviceid);
3739         qemu_drivedel($vmid, $deviceid);
3740         qemu_deletescsihw($conf, $vmid, $deviceid);
3741
3742     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3743
3744         qemu_devicedel($vmid, $deviceid);
3745         qemu_devicedelverify($vmid, $deviceid);
3746         qemu_netdevdel($vmid, $deviceid);
3747
3748     } else {
3749         die "can't unplug device '$deviceid'\n";
3750     }
3751
3752     return 1;
3753 }
3754
3755 sub qemu_deviceadd {
3756     my ($vmid, $devicefull) = @_;
3757
3758     $devicefull = "driver=".$devicefull;
3759     my %options =  split(/[=,]/, $devicefull);
3760
3761     mon_cmd($vmid, "device_add" , %options);
3762 }
3763
3764 sub qemu_devicedel {
3765     my ($vmid, $deviceid) = @_;
3766
3767     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
3768 }
3769
3770 sub qemu_iothread_add {
3771     my($vmid, $deviceid, $device) = @_;
3772
3773     if ($device->{iothread}) {
3774         my $iothreads = vm_iothreads_list($vmid);
3775         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3776     }
3777 }
3778
3779 sub qemu_iothread_del {
3780     my($conf, $vmid, $deviceid) = @_;
3781
3782     my $confid = $deviceid;
3783     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
3784         $confid = 'scsi' . $1;
3785     }
3786     my $device = parse_drive($confid, $conf->{$confid});
3787     if ($device->{iothread}) {
3788         my $iothreads = vm_iothreads_list($vmid);
3789         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3790     }
3791 }
3792
3793 sub qemu_objectadd {
3794     my($vmid, $objectid, $qomtype) = @_;
3795
3796     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3797
3798     return 1;
3799 }
3800
3801 sub qemu_objectdel {
3802     my($vmid, $objectid) = @_;
3803
3804     mon_cmd($vmid, "object-del", id => $objectid);
3805
3806     return 1;
3807 }
3808
3809 sub qemu_driveadd {
3810     my ($storecfg, $vmid, $device) = @_;
3811
3812     my $drive = print_drive_commandline_full($storecfg, $vmid, $device);
3813     $drive =~ s/\\/\\\\/g;
3814     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
3815
3816     # If the command succeeds qemu prints: "OK"
3817     return 1 if $ret =~ m/OK/s;
3818
3819     die "adding drive failed: $ret\n";
3820 }
3821
3822 sub qemu_drivedel {
3823     my($vmid, $deviceid) = @_;
3824
3825     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
3826     $ret =~ s/^\s+//;
3827
3828     return 1 if $ret eq "";
3829
3830     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3831     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3832
3833     die "deleting drive $deviceid failed : $ret\n";
3834 }
3835
3836 sub qemu_deviceaddverify {
3837     my ($vmid, $deviceid) = @_;
3838
3839     for (my $i = 0; $i <= 5; $i++) {
3840          my $devices_list = vm_devices_list($vmid);
3841          return 1 if defined($devices_list->{$deviceid});
3842          sleep 1;
3843     }
3844
3845     die "error on hotplug device '$deviceid'\n";
3846 }
3847
3848
3849 sub qemu_devicedelverify {
3850     my ($vmid, $deviceid) = @_;
3851
3852     # need to verify that the device is correctly removed as device_del
3853     # is async and empty return is not reliable
3854
3855     for (my $i = 0; $i <= 5; $i++) {
3856          my $devices_list = vm_devices_list($vmid);
3857          return 1 if !defined($devices_list->{$deviceid});
3858          sleep 1;
3859     }
3860
3861     die "error on hot-unplugging device '$deviceid'\n";
3862 }
3863
3864 sub qemu_findorcreatescsihw {
3865     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
3866
3867     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3868
3869     my $scsihwid="$controller_prefix$controller";
3870     my $devices_list = vm_devices_list($vmid);
3871
3872     if(!defined($devices_list->{$scsihwid})) {
3873         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
3874     }
3875
3876     return 1;
3877 }
3878
3879 sub qemu_deletescsihw {
3880     my ($conf, $vmid, $opt) = @_;
3881
3882     my $device = parse_drive($opt, $conf->{$opt});
3883
3884     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3885         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3886         return 1;
3887     }
3888
3889     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3890
3891     my $devices_list = vm_devices_list($vmid);
3892     foreach my $opt (keys %{$devices_list}) {
3893         if (is_valid_drivename($opt)) {
3894             my $drive = parse_drive($opt, $conf->{$opt});
3895             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3896                 return 1;
3897             }
3898         }
3899     }
3900
3901     my $scsihwid="scsihw$controller";
3902
3903     vm_deviceunplug($vmid, $conf, $scsihwid);
3904
3905     return 1;
3906 }
3907
3908 sub qemu_add_pci_bridge {
3909     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
3910
3911     my $bridges = {};
3912
3913     my $bridgeid;
3914
3915     print_pci_addr($device, $bridges, $arch, $machine_type);
3916
3917     while (my ($k, $v) = each %$bridges) {
3918         $bridgeid = $k;
3919     }
3920     return 1 if !defined($bridgeid) || $bridgeid < 1;
3921
3922     my $bridge = "pci.$bridgeid";
3923     my $devices_list = vm_devices_list($vmid);
3924
3925     if (!defined($devices_list->{$bridge})) {
3926         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
3927     }
3928
3929     return 1;
3930 }
3931
3932 sub qemu_set_link_status {
3933     my ($vmid, $device, $up) = @_;
3934
3935     mon_cmd($vmid, "set_link", name => $device,
3936                up => $up ? JSON::true : JSON::false);
3937 }
3938
3939 sub qemu_netdevadd {
3940     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
3941
3942     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
3943     my %options =  split(/[=,]/, $netdev);
3944
3945     if (defined(my $vhost = $options{vhost})) {
3946         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
3947     }
3948
3949     if (defined(my $queues = $options{queues})) {
3950         $options{queues} = $queues + 0;
3951     }
3952
3953     mon_cmd($vmid, "netdev_add",  %options);
3954     return 1;
3955 }
3956
3957 sub qemu_netdevdel {
3958     my ($vmid, $deviceid) = @_;
3959
3960     mon_cmd($vmid, "netdev_del", id => $deviceid);
3961 }
3962
3963 sub qemu_usb_hotplug {
3964     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3965
3966     return if !$device;
3967
3968     # remove the old one first
3969     vm_deviceunplug($vmid, $conf, $deviceid);
3970
3971     # check if xhci controller is necessary and available
3972     if ($device->{usb3}) {
3973
3974         my $devicelist = vm_devices_list($vmid);
3975
3976         if (!$devicelist->{xhci}) {
3977             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
3978             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
3979         }
3980     }
3981     my $d = parse_usb_device($device->{host});
3982     $d->{usb3} = $device->{usb3};
3983
3984     # add the new one
3985     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
3986 }
3987
3988 sub qemu_cpu_hotplug {
3989     my ($vmid, $conf, $vcpus) = @_;
3990
3991     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
3992
3993     my $sockets = 1;
3994     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3995     $sockets = $conf->{sockets} if  $conf->{sockets};
3996     my $cores = $conf->{cores} || 1;
3997     my $maxcpus = $sockets * $cores;
3998
3999     $vcpus = $maxcpus if !$vcpus;
4000
4001     die "you can't add more vcpus than maxcpus\n"
4002         if $vcpus > $maxcpus;
4003
4004     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4005
4006     if ($vcpus < $currentvcpus) {
4007
4008         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4009
4010             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4011                 qemu_devicedel($vmid, "cpu$i");
4012                 my $retry = 0;
4013                 my $currentrunningvcpus = undef;
4014                 while (1) {
4015                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4016                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4017                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4018                     $retry++;
4019                     sleep 1;
4020                 }
4021                 #update conf after each succesfull cpu unplug
4022                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4023                 PVE::QemuConfig->write_config($vmid, $conf);
4024             }
4025         } else {
4026             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4027         }
4028
4029         return;
4030     }
4031
4032     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4033     die "vcpus in running vm does not match its configuration\n"
4034         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4035
4036     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4037
4038         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4039             my $cpustr = print_cpu_device($conf, $i);
4040             qemu_deviceadd($vmid, $cpustr);
4041
4042             my $retry = 0;
4043             my $currentrunningvcpus = undef;
4044             while (1) {
4045                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4046                 last if scalar(@{$currentrunningvcpus}) == $i;
4047                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4048                 sleep 1;
4049                 $retry++;
4050             }
4051             #update conf after each succesfull cpu hotplug
4052             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4053             PVE::QemuConfig->write_config($vmid, $conf);
4054         }
4055     } else {
4056
4057         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4058             mon_cmd($vmid, "cpu-add", id => int($i));
4059         }
4060     }
4061 }
4062
4063 sub qemu_block_set_io_throttle {
4064     my ($vmid, $deviceid,
4065         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4066         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4067         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4068         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4069
4070     return if !check_running($vmid) ;
4071
4072     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4073         bps => int($bps),
4074         bps_rd => int($bps_rd),
4075         bps_wr => int($bps_wr),
4076         iops => int($iops),
4077         iops_rd => int($iops_rd),
4078         iops_wr => int($iops_wr),
4079         bps_max => int($bps_max),
4080         bps_rd_max => int($bps_rd_max),
4081         bps_wr_max => int($bps_wr_max),
4082         iops_max => int($iops_max),
4083         iops_rd_max => int($iops_rd_max),
4084         iops_wr_max => int($iops_wr_max),
4085         bps_max_length => int($bps_max_length),
4086         bps_rd_max_length => int($bps_rd_max_length),
4087         bps_wr_max_length => int($bps_wr_max_length),
4088         iops_max_length => int($iops_max_length),
4089         iops_rd_max_length => int($iops_rd_max_length),
4090         iops_wr_max_length => int($iops_wr_max_length),
4091     );
4092
4093 }
4094
4095 # old code, only used to shutdown old VM after update
4096 sub __read_avail {
4097     my ($fh, $timeout) = @_;
4098
4099     my $sel = new IO::Select;
4100     $sel->add($fh);
4101
4102     my $res = '';
4103     my $buf;
4104
4105     my @ready;
4106     while (scalar (@ready = $sel->can_read($timeout))) {
4107         my $count;
4108         if ($count = $fh->sysread($buf, 8192)) {
4109             if ($buf =~ /^(.*)\(qemu\) $/s) {
4110                 $res .= $1;
4111                 last;
4112             } else {
4113                 $res .= $buf;
4114             }
4115         } else {
4116             if (!defined($count)) {
4117                 die "$!\n";
4118             }
4119             last;
4120         }
4121     }
4122
4123     die "monitor read timeout\n" if !scalar(@ready);
4124
4125     return $res;
4126 }
4127
4128 sub qemu_block_resize {
4129     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4130
4131     my $running = check_running($vmid);
4132
4133     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4134
4135     return if !$running;
4136
4137     my $padding = (1024 - $size % 1024) % 1024;
4138     $size = $size + $padding;
4139
4140     mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4141
4142 }
4143
4144 sub qemu_volume_snapshot {
4145     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4146
4147     my $running = check_running($vmid);
4148
4149     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4150         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4151     } else {
4152         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4153     }
4154 }
4155
4156 sub qemu_volume_snapshot_delete {
4157     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4158
4159     my $running = check_running($vmid);
4160
4161     if($running) {
4162
4163         $running = undef;
4164         my $conf = PVE::QemuConfig->load_config($vmid);
4165         PVE::QemuConfig->foreach_volume($conf, sub {
4166             my ($ds, $drive) = @_;
4167             $running = 1 if $drive->{file} eq $volid;
4168         });
4169     }
4170
4171     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4172         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4173     } else {
4174         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4175     }
4176 }
4177
4178 sub set_migration_caps {
4179     my ($vmid) = @_;
4180
4181     my $cap_ref = [];
4182
4183     my $enabled_cap = {
4184         "auto-converge" => 1,
4185         "xbzrle" => 1,
4186         "x-rdma-pin-all" => 0,
4187         "zero-blocks" => 0,
4188         "compress" => 0
4189     };
4190
4191     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4192
4193     for my $supported_capability (@$supported_capabilities) {
4194         push @$cap_ref, {
4195             capability => $supported_capability->{capability},
4196             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4197         };
4198     }
4199
4200     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4201 }
4202
4203 sub foreach_volid {
4204     my ($conf, $func, @param) = @_;
4205
4206     my $volhash = {};
4207
4208     my $test_volid = sub {
4209         my ($key, $drive, $snapname) = @_;
4210
4211         my $volid = $drive->{file};
4212         return if !$volid;
4213
4214         $volhash->{$volid}->{cdrom} //= 1;
4215         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4216
4217         my $replicate = $drive->{replicate} // 1;
4218         $volhash->{$volid}->{replicate} //= 0;
4219         $volhash->{$volid}->{replicate} = 1 if $replicate;
4220
4221         $volhash->{$volid}->{shared} //= 0;
4222         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4223
4224         $volhash->{$volid}->{referenced_in_config} //= 0;
4225         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4226
4227         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4228             if defined($snapname);
4229
4230         my $size = $drive->{size};
4231         $volhash->{$volid}->{size} //= $size if $size;
4232
4233         $volhash->{$volid}->{is_vmstate} //= 0;
4234         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4235
4236         $volhash->{$volid}->{is_unused} //= 0;
4237         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4238     };
4239
4240     my $include_opts = {
4241         extra_keys => ['vmstate'],
4242         include_unused => 1,
4243     };
4244
4245     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4246     foreach my $snapname (keys %{$conf->{snapshots}}) {
4247         my $snap = $conf->{snapshots}->{$snapname};
4248         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4249     }
4250
4251     foreach my $volid (keys %$volhash) {
4252         &$func($volid, $volhash->{$volid}, @param);
4253     }
4254 }
4255
4256 my $fast_plug_option = {
4257     'lock' => 1,
4258     'name' => 1,
4259     'onboot' => 1,
4260     'shares' => 1,
4261     'startup' => 1,
4262     'description' => 1,
4263     'protection' => 1,
4264     'vmstatestorage' => 1,
4265     'hookscript' => 1,
4266     'tags' => 1,
4267 };
4268
4269 # hotplug changes in [PENDING]
4270 # $selection hash can be used to only apply specified options, for
4271 # example: { cores => 1 } (only apply changed 'cores')
4272 # $errors ref is used to return error messages
4273 sub vmconfig_hotplug_pending {
4274     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4275
4276     my $defaults = load_defaults();
4277     my $arch = get_vm_arch($conf);
4278     my $machine_type = get_vm_machine($conf, undef, $arch);
4279
4280     # commit values which do not have any impact on running VM first
4281     # Note: those option cannot raise errors, we we do not care about
4282     # $selection and always apply them.
4283
4284     my $add_error = sub {
4285         my ($opt, $msg) = @_;
4286         $errors->{$opt} = "hotplug problem - $msg";
4287     };
4288
4289     my $changes = 0;
4290     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4291         if ($fast_plug_option->{$opt}) {
4292             $conf->{$opt} = $conf->{pending}->{$opt};
4293             delete $conf->{pending}->{$opt};
4294             $changes = 1;
4295         }
4296     }
4297
4298     if ($changes) {
4299         PVE::QemuConfig->write_config($vmid, $conf);
4300     }
4301
4302     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4303
4304     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4305     foreach my $opt (sort keys %$pending_delete_hash) {
4306         next if $selection && !$selection->{$opt};
4307         my $force = $pending_delete_hash->{$opt}->{force};
4308         eval {
4309             if ($opt eq 'hotplug') {
4310                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4311             } elsif ($opt eq 'tablet') {
4312                 die "skip\n" if !$hotplug_features->{usb};
4313                 if ($defaults->{tablet}) {
4314                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4315                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4316                         if $arch eq 'aarch64';
4317                 } else {
4318                     vm_deviceunplug($vmid, $conf, 'tablet');
4319                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4320                 }
4321             } elsif ($opt =~ m/^usb\d+/) {
4322                 die "skip\n";
4323                 # since we cannot reliably hot unplug usb devices
4324                 # we are disabling it
4325                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4326                 vm_deviceunplug($vmid, $conf, $opt);
4327             } elsif ($opt eq 'vcpus') {
4328                 die "skip\n" if !$hotplug_features->{cpu};
4329                 qemu_cpu_hotplug($vmid, $conf, undef);
4330             } elsif ($opt eq 'balloon') {
4331                 # enable balloon device is not hotpluggable
4332                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4333                 # here we reset the ballooning value to memory
4334                 my $balloon = $conf->{memory} || $defaults->{memory};
4335                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4336             } elsif ($fast_plug_option->{$opt}) {
4337                 # do nothing
4338             } elsif ($opt =~ m/^net(\d+)$/) {
4339                 die "skip\n" if !$hotplug_features->{network};
4340                 vm_deviceunplug($vmid, $conf, $opt);
4341             } elsif (is_valid_drivename($opt)) {
4342                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4343                 vm_deviceunplug($vmid, $conf, $opt);
4344                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4345             } elsif ($opt =~ m/^memory$/) {
4346                 die "skip\n" if !$hotplug_features->{memory};
4347                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4348             } elsif ($opt eq 'cpuunits') {
4349                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4350             } elsif ($opt eq 'cpulimit') {
4351                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4352             } else {
4353                 die "skip\n";
4354             }
4355         };
4356         if (my $err = $@) {
4357             &$add_error($opt, $err) if $err ne "skip\n";
4358         } else {
4359             delete $conf->{$opt};
4360             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4361         }
4362     }
4363
4364     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4365     $apply_pending_cloudinit = sub {
4366         return if $apply_pending_cloudinit_done; # once is enough
4367         $apply_pending_cloudinit_done = 1; # once is enough
4368
4369         my ($key, $value) = @_;
4370
4371         my @cloudinit_opts = keys %$confdesc_cloudinit;
4372         foreach my $opt (keys %{$conf->{pending}}) {
4373             next if !grep { $_ eq $opt } @cloudinit_opts;
4374             $conf->{$opt} = delete $conf->{pending}->{$opt};
4375         }
4376
4377         my $new_conf = { %$conf };
4378         $new_conf->{$key} = $value;
4379         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4380     };
4381
4382     foreach my $opt (keys %{$conf->{pending}}) {
4383         next if $selection && !$selection->{$opt};
4384         my $value = $conf->{pending}->{$opt};
4385         eval {
4386             if ($opt eq 'hotplug') {
4387                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4388             } elsif ($opt eq 'tablet') {
4389                 die "skip\n" if !$hotplug_features->{usb};
4390                 if ($value == 1) {
4391                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4392                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4393                         if $arch eq 'aarch64';
4394                 } elsif ($value == 0) {
4395                     vm_deviceunplug($vmid, $conf, 'tablet');
4396                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4397                 }
4398             } elsif ($opt =~ m/^usb\d+$/) {
4399                 die "skip\n";
4400                 # since we cannot reliably hot unplug usb devices
4401                 # we are disabling it
4402                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4403                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4404                 die "skip\n" if !$d;
4405                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4406             } elsif ($opt eq 'vcpus') {
4407                 die "skip\n" if !$hotplug_features->{cpu};
4408                 qemu_cpu_hotplug($vmid, $conf, $value);
4409             } elsif ($opt eq 'balloon') {
4410                 # enable/disable balloning device is not hotpluggable
4411                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4412                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4413                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4414
4415                 # allow manual ballooning if shares is set to zero
4416                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4417                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4418                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4419                 }
4420             } elsif ($opt =~ m/^net(\d+)$/) {
4421                 # some changes can be done without hotplug
4422                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4423                                     $vmid, $opt, $value, $arch, $machine_type);
4424             } elsif (is_valid_drivename($opt)) {
4425                 die "skip\n" if $opt eq 'efidisk0';
4426                 # some changes can be done without hotplug
4427                 my $drive = parse_drive($opt, $value);
4428                 if (drive_is_cloudinit($drive)) {
4429                     &$apply_pending_cloudinit($opt, $value);
4430                 }
4431                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4432                                      $vmid, $opt, $value, $arch, $machine_type);
4433             } elsif ($opt =~ m/^memory$/) { #dimms
4434                 die "skip\n" if !$hotplug_features->{memory};
4435                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4436             } elsif ($opt eq 'cpuunits') {
4437                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4438             } elsif ($opt eq 'cpulimit') {
4439                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4440                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4441             } else {
4442                 die "skip\n";  # skip non-hot-pluggable options
4443             }
4444         };
4445         if (my $err = $@) {
4446             &$add_error($opt, $err) if $err ne "skip\n";
4447         } else {
4448             $conf->{$opt} = $value;
4449             delete $conf->{pending}->{$opt};
4450         }
4451     }
4452
4453     PVE::QemuConfig->write_config($vmid, $conf);
4454 }
4455
4456 sub try_deallocate_drive {
4457     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4458
4459     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4460         my $volid = $drive->{file};
4461         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4462             my $sid = PVE::Storage::parse_volume_id($volid);
4463             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4464
4465             # check if the disk is really unused
4466             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4467                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4468             PVE::Storage::vdisk_free($storecfg, $volid);
4469             return 1;
4470         } else {
4471             # If vm is not owner of this disk remove from config
4472             return 1;
4473         }
4474     }
4475
4476     return undef;
4477 }
4478
4479 sub vmconfig_delete_or_detach_drive {
4480     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4481
4482     my $drive = parse_drive($opt, $conf->{$opt});
4483
4484     my $rpcenv = PVE::RPCEnvironment::get();
4485     my $authuser = $rpcenv->get_user();
4486
4487     if ($force) {
4488         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4489         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4490     } else {
4491         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4492     }
4493 }
4494
4495
4496
4497 sub vmconfig_apply_pending {
4498     my ($vmid, $conf, $storecfg, $errors) = @_;
4499
4500     my $add_apply_error = sub {
4501         my ($opt, $msg) = @_;
4502         my $err_msg = "unable to apply pending change $opt : $msg";
4503         $errors->{$opt} = $err_msg;
4504         warn $err_msg;
4505     };
4506
4507     # cold plug
4508
4509     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4510     foreach my $opt (sort keys %$pending_delete_hash) {
4511         my $force = $pending_delete_hash->{$opt}->{force};
4512         eval {
4513             if ($opt =~ m/^unused/) {
4514                 die "internal error";
4515             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4516                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4517             }
4518         };
4519         if (my $err = $@) {
4520             $add_apply_error->($opt, $err);
4521         } else {
4522             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4523             delete $conf->{$opt};
4524         }
4525     }
4526
4527     PVE::QemuConfig->cleanup_pending($conf);
4528
4529     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4530         next if $opt eq 'delete'; # just to be sure
4531         eval {
4532             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4533                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4534             }
4535         };
4536         if (my $err = $@) {
4537             $add_apply_error->($opt, $err);
4538         } else {
4539             $conf->{$opt} = delete $conf->{pending}->{$opt};
4540         }
4541     }
4542
4543     # write all changes at once to avoid unnecessary i/o
4544     PVE::QemuConfig->write_config($vmid, $conf);
4545 }
4546
4547 sub vmconfig_update_net {
4548     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4549
4550     my $newnet = parse_net($value);
4551
4552     if ($conf->{$opt}) {
4553         my $oldnet = parse_net($conf->{$opt});
4554
4555         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4556             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4557             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4558             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4559
4560             # for non online change, we try to hot-unplug
4561             die "skip\n" if !$hotplug;
4562             vm_deviceunplug($vmid, $conf, $opt);
4563         } else {
4564
4565             die "internal error" if $opt !~ m/net(\d+)/;
4566             my $iface = "tap${vmid}i$1";
4567
4568             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4569                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4570                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4571                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4572                 PVE::Network::tap_unplug($iface);
4573
4574                 if ($have_sdn) {
4575                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4576                 } else {
4577                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4578                 }
4579             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4580                 # Rate can be applied on its own but any change above needs to
4581                 # include the rate in tap_plug since OVS resets everything.
4582                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4583             }
4584
4585             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4586                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4587             }
4588
4589             return 1;
4590         }
4591     }
4592
4593     if ($hotplug) {
4594         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4595     } else {
4596         die "skip\n";
4597     }
4598 }
4599
4600 sub vmconfig_update_disk {
4601     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4602
4603     my $drive = parse_drive($opt, $value);
4604
4605     if ($conf->{$opt}) {
4606
4607         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4608
4609             my $media = $drive->{media} || 'disk';
4610             my $oldmedia = $old_drive->{media} || 'disk';
4611             die "unable to change media type\n" if $media ne $oldmedia;
4612
4613             if (!drive_is_cdrom($old_drive)) {
4614
4615                 if ($drive->{file} ne $old_drive->{file}) {
4616
4617                     die "skip\n" if !$hotplug;
4618
4619                     # unplug and register as unused
4620                     vm_deviceunplug($vmid, $conf, $opt);
4621                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4622
4623                 } else {
4624                     # update existing disk
4625
4626                     # skip non hotpluggable value
4627                     if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4628                         safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4629                         safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4630                         safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
4631                         safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
4632                         die "skip\n";
4633                     }
4634
4635                     # apply throttle
4636                     if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4637                         safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4638                         safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4639                         safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4640                         safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4641                         safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4642                         safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4643                         safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4644                         safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4645                         safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4646                         safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4647                         safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4648                         safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4649                         safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4650                         safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4651                         safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4652                         safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4653                         safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4654
4655                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4656                                                    ($drive->{mbps} || 0)*1024*1024,
4657                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4658                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4659                                                    $drive->{iops} || 0,
4660                                                    $drive->{iops_rd} || 0,
4661                                                    $drive->{iops_wr} || 0,
4662                                                    ($drive->{mbps_max} || 0)*1024*1024,
4663                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4664                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4665                                                    $drive->{iops_max} || 0,
4666                                                    $drive->{iops_rd_max} || 0,
4667                                                    $drive->{iops_wr_max} || 0,
4668                                                    $drive->{bps_max_length} || 1,
4669                                                    $drive->{bps_rd_max_length} || 1,
4670                                                    $drive->{bps_wr_max_length} || 1,
4671                                                    $drive->{iops_max_length} || 1,
4672                                                    $drive->{iops_rd_max_length} || 1,
4673                                                    $drive->{iops_wr_max_length} || 1);
4674
4675                     }
4676
4677                     return 1;
4678                 }
4679
4680             } else { # cdrom
4681
4682                 if ($drive->{file} eq 'none') {
4683                     mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4684                     if (drive_is_cloudinit($old_drive)) {
4685                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4686                     }
4687                 } else {
4688                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4689
4690                     # force eject if locked
4691                     mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4692
4693                     if ($path) {
4694                         mon_cmd($vmid, "blockdev-change-medium",
4695                             id => "$opt", filename => "$path");
4696                     }
4697                 }
4698
4699                 return 1;
4700             }
4701         }
4702     }
4703
4704     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4705     # hotplug new disks
4706     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4707     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
4708 }
4709
4710 # called in locked context by incoming migration
4711 sub vm_migrate_get_nbd_disks {
4712     my ($storecfg, $conf, $replicated_volumes) = @_;
4713
4714     my $local_volumes = {};
4715     PVE::QemuConfig->foreach_volume($conf, sub {
4716         my ($ds, $drive) = @_;
4717
4718         return if drive_is_cdrom($drive);
4719
4720         my $volid = $drive->{file};
4721
4722         return if !$volid;
4723
4724         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4725
4726         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4727         return if $scfg->{shared};
4728
4729         # replicated disks re-use existing state via bitmap
4730         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
4731         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
4732     });
4733     return $local_volumes;
4734 }
4735
4736 # called in locked context by incoming migration
4737 sub vm_migrate_alloc_nbd_disks {
4738     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
4739
4740     my $format = undef;
4741
4742     my $nbd = {};
4743     foreach my $opt (sort keys %$source_volumes) {
4744         my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
4745
4746         if ($use_existing) {
4747             $nbd->{$opt}->{drivestr} = print_drive($drive);
4748             $nbd->{$opt}->{volid} = $volid;
4749             $nbd->{$opt}->{replicated} = 1;
4750             next;
4751         }
4752
4753         # If a remote storage is specified and the format of the original
4754         # volume is not available there, fall back to the default format.
4755         # Otherwise use the same format as the original.
4756         if (!$storagemap->{identity}) {
4757             $storeid = map_storage($storagemap, $storeid);
4758             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4759             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4760             my $fileFormat = qemu_img_format($scfg, $volname);
4761             $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
4762         } else {
4763             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4764             $format = qemu_img_format($scfg, $volname);
4765         }
4766
4767         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4768         my $newdrive = $drive;
4769         $newdrive->{format} = $format;
4770         $newdrive->{file} = $newvolid;
4771         my $drivestr = print_drive($newdrive);
4772         $nbd->{$opt}->{drivestr} = $drivestr;
4773         $nbd->{$opt}->{volid} = $newvolid;
4774     }
4775
4776     return $nbd;
4777 }
4778
4779 # see vm_start_nolock for parameters, additionally:
4780 # migrate_opts:
4781 #   storagemap = parsed storage map for allocating NBD disks
4782 sub vm_start {
4783     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
4784
4785     return PVE::QemuConfig->lock_config($vmid, sub {
4786         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
4787
4788         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4789
4790         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
4791
4792         PVE::QemuConfig->check_lock($conf)
4793             if !($params->{skiplock} || $has_suspended_lock);
4794
4795         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
4796
4797         die "VM $vmid already running\n" if check_running($vmid, undef, $migrate_opts->{migratedfrom});
4798
4799         if (my $storagemap = $migrate_opts->{storagemap}) {
4800             my $replicated = $migrate_opts->{replicated_volumes};
4801             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
4802             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
4803
4804             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
4805                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
4806             }
4807         }
4808
4809         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
4810     });
4811 }
4812
4813
4814 # params:
4815 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
4816 #   skiplock => 0/1, skip checking for config lock
4817 #   forcemachine => to force Qemu machine (rollback/migration)
4818 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
4819 #   timeout => in seconds
4820 #   paused => start VM in paused state (backup)
4821 #   resume => resume from hibernation
4822 # migrate_opts:
4823 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
4824 #   migratedfrom => source node
4825 #   spice_ticket => used for spice migration, passed via tunnel/stdin
4826 #   network => CIDR of migration network
4827 #   type => secure/insecure - tunnel over encrypted connection or plain-text
4828 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
4829 #   replicated_volumes = which volids should be re-used with bitmaps for nbd migration
4830 sub vm_start_nolock {
4831     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
4832
4833     my $statefile = $params->{statefile};
4834     my $resume = $params->{resume};
4835
4836     my $migratedfrom = $migrate_opts->{migratedfrom};
4837     my $migration_type = $migrate_opts->{type};
4838
4839     my $res = {};
4840
4841     # clean up leftover reboot request files
4842     eval { clear_reboot_request($vmid); };
4843     warn $@ if $@;
4844
4845     if (!$statefile && scalar(keys %{$conf->{pending}})) {
4846         vmconfig_apply_pending($vmid, $conf, $storecfg);
4847         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4848     }
4849
4850     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4851
4852     my $defaults = load_defaults();
4853
4854     # set environment variable useful inside network script
4855     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4856
4857     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
4858
4859     my $forcemachine = $params->{forcemachine};
4860     my $forcecpu = $params->{forcecpu};
4861     if ($resume) {
4862         # enforce machine and CPU type on suspended vm to ensure HW compatibility
4863         $forcemachine = $conf->{runningmachine};
4864         $forcecpu = $conf->{runningcpu};
4865         print "Resuming suspended VM\n";
4866     }
4867
4868     my ($cmd, $vollist, $spice_port) =
4869         config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
4870
4871     my $migration_ip;
4872     my $get_migration_ip = sub {
4873         my ($nodename) = @_;
4874
4875         return $migration_ip if defined($migration_ip);
4876
4877         my $cidr = $migrate_opts->{network};
4878
4879         if (!defined($cidr)) {
4880             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4881             $cidr = $dc_conf->{migration}->{network};
4882         }
4883
4884         if (defined($cidr)) {
4885             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
4886
4887             die "could not get IP: no address configured on local " .
4888                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
4889
4890             die "could not get IP: multiple addresses configured on local " .
4891                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
4892
4893             $migration_ip = @$ips[0];
4894         }
4895
4896         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
4897             if !defined($migration_ip);
4898
4899         return $migration_ip;
4900     };
4901
4902     my $migrate_uri;
4903     if ($statefile) {
4904         if ($statefile eq 'tcp') {
4905             my $localip = "localhost";
4906             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4907             my $nodename = nodename();
4908
4909             if (!defined($migration_type)) {
4910                 if (defined($datacenterconf->{migration}->{type})) {
4911                     $migration_type = $datacenterconf->{migration}->{type};
4912                 } else {
4913                     $migration_type = 'secure';
4914                 }
4915             }
4916
4917             if ($migration_type eq 'insecure') {
4918                 $localip = $get_migration_ip->($nodename);
4919                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4920             }
4921
4922             my $pfamily = PVE::Tools::get_host_address_family($nodename);
4923             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4924             $migrate_uri = "tcp:${localip}:${migrate_port}";
4925             push @$cmd, '-incoming', $migrate_uri;
4926             push @$cmd, '-S';
4927
4928         } elsif ($statefile eq 'unix') {
4929             # should be default for secure migrations as a ssh TCP forward
4930             # tunnel is not deterministic reliable ready and fails regurarly
4931             # to set up in time, so use UNIX socket forwards
4932             my $socket_addr = "/run/qemu-server/$vmid.migrate";
4933             unlink $socket_addr;
4934
4935             $migrate_uri = "unix:$socket_addr";
4936
4937             push @$cmd, '-incoming', $migrate_uri;
4938             push @$cmd, '-S';
4939
4940         } elsif (-e $statefile) {
4941             push @$cmd, '-loadstate', $statefile;
4942         } else {
4943             my $statepath = PVE::Storage::path($storecfg, $statefile);
4944             push @$vollist, $statefile;
4945             push @$cmd, '-loadstate', $statepath;
4946         }
4947     } elsif ($params->{paused}) {
4948         push @$cmd, '-S';
4949     }
4950
4951     # host pci devices
4952     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
4953       my $d = parse_hostpci($conf->{"hostpci$i"});
4954       next if !$d;
4955       my $pcidevices = $d->{pciid};
4956       foreach my $pcidevice (@$pcidevices) {
4957             my $pciid = $pcidevice->{id};
4958
4959             my $info = PVE::SysFSTools::pci_device_info("$pciid");
4960             die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
4961             die "no pci device info for device '$pciid'\n" if !$info;
4962
4963             if ($d->{mdev}) {
4964                 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
4965                 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
4966             } else {
4967                 die "can't unbind/bind pci group to vfio '$pciid'\n"
4968                     if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
4969                 die "can't reset pci device '$pciid'\n"
4970                     if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
4971             }
4972       }
4973     }
4974
4975     PVE::Storage::activate_volumes($storecfg, $vollist);
4976
4977     eval {
4978         run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
4979             outfunc => sub {}, errfunc => sub {});
4980     };
4981     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
4982     # timeout should be more than enough here...
4983     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
4984
4985     my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4986                                               : $defaults->{cpuunits};
4987
4988     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
4989     my %run_params = (
4990         timeout => $statefile ? undef : $start_timeout,
4991         umask => 0077,
4992         noerr => 1,
4993     );
4994
4995     # when migrating, prefix QEMU output so other side can pick up any
4996     # errors that might occur and show the user
4997     if ($migratedfrom) {
4998         $run_params{quiet} = 1;
4999         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5000     }
5001
5002     my %properties = (
5003         Slice => 'qemu.slice',
5004         KillMode => 'none',
5005         CPUShares => $cpuunits
5006     );
5007
5008     if (my $cpulimit = $conf->{cpulimit}) {
5009         $properties{CPUQuota} = int($cpulimit * 100);
5010     }
5011     $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5012
5013     my $run_qemu = sub {
5014         PVE::Tools::run_fork sub {
5015             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5016
5017             my $exitcode = run_command($cmd, %run_params);
5018             die "QEMU exited with code $exitcode\n" if $exitcode;
5019         };
5020     };
5021
5022     if ($conf->{hugepages}) {
5023
5024         my $code = sub {
5025             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5026             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5027
5028             PVE::QemuServer::Memory::hugepages_mount();
5029             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5030
5031             eval { $run_qemu->() };
5032             if (my $err = $@) {
5033                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5034                 die $err;
5035             }
5036
5037             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5038         };
5039         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5040
5041     } else {
5042         eval { $run_qemu->() };
5043     }
5044
5045     if (my $err = $@) {
5046         # deactivate volumes if start fails
5047         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5048         die "start failed: $err";
5049     }
5050
5051     print "migration listens on $migrate_uri\n" if $migrate_uri;
5052     $res->{migrate_uri} = $migrate_uri;
5053
5054     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5055         eval { mon_cmd($vmid, "cont"); };
5056         warn $@ if $@;
5057     }
5058
5059     #start nbd server for storage migration
5060     if (my $nbd = $migrate_opts->{nbd}) {
5061         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5062
5063         my $migrate_storage_uri;
5064         # nbd_protocol_version > 0 for unix socket support
5065         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5066             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5067             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5068             $migrate_storage_uri = "nbd:unix:$socket_path";
5069         } else {
5070             my $nodename = nodename();
5071             my $localip = $get_migration_ip->($nodename);
5072             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5073             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5074
5075             mon_cmd($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${storage_migrate_port}" } } );
5076             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5077             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5078         }
5079
5080         $res->{migrate_storage_uri} = $migrate_storage_uri;
5081
5082         foreach my $opt (sort keys %$nbd) {
5083             my $drivestr = $nbd->{$opt}->{drivestr};
5084             my $volid = $nbd->{$opt}->{volid};
5085             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5086             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5087             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5088             print "re-using replicated volume: $opt - $volid\n"
5089                 if $nbd->{$opt}->{replicated};
5090
5091             $res->{drives}->{$opt} = $nbd->{$opt};
5092             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5093         }
5094     }
5095
5096     if ($migratedfrom) {
5097         eval {
5098             set_migration_caps($vmid);
5099         };
5100         warn $@ if $@;
5101
5102         if ($spice_port) {
5103             print "spice listens on port $spice_port\n";
5104             $res->{spice_port} = $spice_port;
5105             if ($migrate_opts->{spice_ticket}) {
5106                 mon_cmd($vmid, "set_password", protocol => 'spice', password => $migrate_opts->{spice_ticket});
5107                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5108             }
5109         }
5110
5111     } else {
5112         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5113             if !$statefile && $conf->{balloon};
5114
5115         foreach my $opt (keys %$conf) {
5116             next if $opt !~  m/^net\d+$/;
5117             my $nicconf = parse_net($conf->{$opt});
5118             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5119         }
5120     }
5121
5122     mon_cmd($vmid, 'qom-set',
5123                 path => "machine/peripheral/balloon0",
5124                 property => "guest-stats-polling-interval",
5125                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5126
5127     if ($resume) {
5128         print "Resumed VM, removing state\n";
5129         if (my $vmstate = $conf->{vmstate}) {
5130             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5131             PVE::Storage::vdisk_free($storecfg, $vmstate);
5132         }
5133         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5134         PVE::QemuConfig->write_config($vmid, $conf);
5135     }
5136
5137     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5138
5139     return $res;
5140 }
5141
5142 sub vm_commandline {
5143     my ($storecfg, $vmid, $snapname) = @_;
5144
5145     my $conf = PVE::QemuConfig->load_config($vmid);
5146     my $forcemachine;
5147     my $forcecpu;
5148
5149     if ($snapname) {
5150         my $snapshot = $conf->{snapshots}->{$snapname};
5151         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5152
5153         # check for machine or CPU overrides in snapshot
5154         $forcemachine = $snapshot->{runningmachine};
5155         $forcecpu = $snapshot->{runningcpu};
5156
5157         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5158
5159         $conf = $snapshot;
5160     }
5161
5162     my $defaults = load_defaults();
5163
5164     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults,
5165         $forcemachine, $forcecpu);
5166
5167     return PVE::Tools::cmd2string($cmd);
5168 }
5169
5170 sub vm_reset {
5171     my ($vmid, $skiplock) = @_;
5172
5173     PVE::QemuConfig->lock_config($vmid, sub {
5174
5175         my $conf = PVE::QemuConfig->load_config($vmid);
5176
5177         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5178
5179         mon_cmd($vmid, "system_reset");
5180     });
5181 }
5182
5183 sub get_vm_volumes {
5184     my ($conf) = @_;
5185
5186     my $vollist = [];
5187     foreach_volid($conf, sub {
5188         my ($volid, $attr) = @_;
5189
5190         return if $volid =~ m|^/|;
5191
5192         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5193         return if !$sid;
5194
5195         push @$vollist, $volid;
5196     });
5197
5198     return $vollist;
5199 }
5200
5201 sub vm_stop_cleanup {
5202     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5203
5204     eval {
5205
5206         if (!$keepActive) {
5207             my $vollist = get_vm_volumes($conf);
5208             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5209         }
5210
5211         foreach my $ext (qw(mon qmp pid vnc qga)) {
5212             unlink "/var/run/qemu-server/${vmid}.$ext";
5213         }
5214
5215         if ($conf->{ivshmem}) {
5216             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5217             # just delete it for now, VMs which have this already open do not
5218             # are affected, but new VMs will get a separated one. If this
5219             # becomes an issue we either add some sort of ref-counting or just
5220             # add a "don't delete on stop" flag to the ivshmem format.
5221             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5222         }
5223
5224         foreach my $key (keys %$conf) {
5225             next if $key !~ m/^hostpci(\d+)$/;
5226             my $hostpciindex = $1;
5227             my $d = parse_hostpci($conf->{$key});
5228             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5229
5230             foreach my $pci (@{$d->{pciid}}) {
5231                 my $pciid = $pci->{id};
5232                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5233             }
5234         }
5235
5236         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5237     };
5238     warn $@ if $@; # avoid errors - just warn
5239 }
5240
5241 # call only in locked context
5242 sub _do_vm_stop {
5243     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5244
5245     my $pid = check_running($vmid, $nocheck);
5246     return if !$pid;
5247
5248     my $conf;
5249     if (!$nocheck) {
5250         $conf = PVE::QemuConfig->load_config($vmid);
5251         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5252         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5253             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5254             $timeout = $opts->{down} if $opts->{down};
5255         }
5256         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5257     }
5258
5259     eval {
5260         if ($shutdown) {
5261             if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5262                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5263             } else {
5264                 mon_cmd($vmid, "system_powerdown");
5265             }
5266         } else {
5267             mon_cmd($vmid, "quit");
5268         }
5269     };
5270     my $err = $@;
5271
5272     if (!$err) {
5273         $timeout = 60 if !defined($timeout);
5274
5275         my $count = 0;
5276         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5277             $count++;
5278             sleep 1;
5279         }
5280
5281         if ($count >= $timeout) {
5282             if ($force) {
5283                 warn "VM still running - terminating now with SIGTERM\n";
5284                 kill 15, $pid;
5285             } else {
5286                 die "VM quit/powerdown failed - got timeout\n";
5287             }
5288         } else {
5289             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5290             return;
5291         }
5292     } else {
5293        if (!check_running($vmid, $nocheck)) {
5294             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5295             return;
5296        }
5297        if ($force) {
5298             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5299             kill 15, $pid;
5300         } else {
5301             die "VM quit/powerdown failed\n";
5302         }
5303     }
5304
5305     # wait again
5306     $timeout = 10;
5307
5308     my $count = 0;
5309     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5310         $count++;
5311         sleep 1;
5312     }
5313
5314     if ($count >= $timeout) {
5315         warn "VM still running - terminating now with SIGKILL\n";
5316         kill 9, $pid;
5317         sleep 1;
5318     }
5319
5320     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5321 }
5322
5323 # Note: use $nocheck to skip tests if VM configuration file exists.
5324 # We need that when migration VMs to other nodes (files already moved)
5325 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5326 sub vm_stop {
5327     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5328
5329     $force = 1 if !defined($force) && !$shutdown;
5330
5331     if ($migratedfrom){
5332         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5333         kill 15, $pid if $pid;
5334         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5335         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5336         return;
5337     }
5338
5339     PVE::QemuConfig->lock_config($vmid, sub {
5340         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5341    });
5342 }
5343
5344 sub vm_reboot {
5345     my ($vmid, $timeout) = @_;
5346
5347     PVE::QemuConfig->lock_config($vmid, sub {
5348         eval {
5349
5350             # only reboot if running, as qmeventd starts it again on a stop event
5351             return if !check_running($vmid);
5352
5353             create_reboot_request($vmid);
5354
5355             my $storecfg = PVE::Storage::config();
5356             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5357
5358         };
5359         if (my $err = $@) {
5360             # avoid that the next normal shutdown will be confused for a reboot
5361             clear_reboot_request($vmid);
5362             die $err;
5363         }
5364    });
5365 }
5366
5367 # note: if using the statestorage parameter, the caller has to check privileges
5368 sub vm_suspend {
5369     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5370
5371     my $conf;
5372     my $path;
5373     my $storecfg;
5374     my $vmstate;
5375
5376     PVE::QemuConfig->lock_config($vmid, sub {
5377
5378         $conf = PVE::QemuConfig->load_config($vmid);
5379
5380         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5381         PVE::QemuConfig->check_lock($conf)
5382             if !($skiplock || $is_backing_up);
5383
5384         die "cannot suspend to disk during backup\n"
5385             if $is_backing_up && $includestate;
5386
5387         if ($includestate) {
5388             $conf->{lock} = 'suspending';
5389             my $date = strftime("%Y-%m-%d", localtime(time()));
5390             $storecfg = PVE::Storage::config();
5391             if (!$statestorage) {
5392                 $statestorage = find_vmstate_storage($conf, $storecfg);
5393                 # check permissions for the storage
5394                 my $rpcenv = PVE::RPCEnvironment::get();
5395                 if ($rpcenv->{type} ne 'cli') {
5396                     my $authuser = $rpcenv->get_user();
5397                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5398                 }
5399             }
5400
5401
5402             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5403             $path = PVE::Storage::path($storecfg, $vmstate);
5404             PVE::QemuConfig->write_config($vmid, $conf);
5405         } else {
5406             mon_cmd($vmid, "stop");
5407         }
5408     });
5409
5410     if ($includestate) {
5411         # save vm state
5412         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5413
5414         eval {
5415             mon_cmd($vmid, "savevm-start", statefile => $path);
5416             for(;;) {
5417                 my $state = mon_cmd($vmid, "query-savevm");
5418                 if (!$state->{status}) {
5419                     die "savevm not active\n";
5420                 } elsif ($state->{status} eq 'active') {
5421                     sleep(1);
5422                     next;
5423                 } elsif ($state->{status} eq 'completed') {
5424                     print "State saved, quitting\n";
5425                     last;
5426                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5427                     die "query-savevm failed with error '$state->{error}'\n"
5428                 } else {
5429                     die "query-savevm returned status '$state->{status}'\n";
5430                 }
5431             }
5432         };
5433         my $err = $@;
5434
5435         PVE::QemuConfig->lock_config($vmid, sub {
5436             $conf = PVE::QemuConfig->load_config($vmid);
5437             if ($err) {
5438                 # cleanup, but leave suspending lock, to indicate something went wrong
5439                 eval {
5440                     mon_cmd($vmid, "savevm-end");
5441                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5442                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5443                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
5444                     PVE::QemuConfig->write_config($vmid, $conf);
5445                 };
5446                 warn $@ if $@;
5447                 die $err;
5448             }
5449
5450             die "lock changed unexpectedly\n"
5451                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5452
5453             mon_cmd($vmid, "quit");
5454             $conf->{lock} = 'suspended';
5455             PVE::QemuConfig->write_config($vmid, $conf);
5456         });
5457     }
5458 }
5459
5460 sub vm_resume {
5461     my ($vmid, $skiplock, $nocheck) = @_;
5462
5463     PVE::QemuConfig->lock_config($vmid, sub {
5464         my $res = mon_cmd($vmid, 'query-status');
5465         my $resume_cmd = 'cont';
5466
5467         if ($res->{status} && $res->{status} eq 'suspended') {
5468             $resume_cmd = 'system_wakeup';
5469         }
5470
5471         if (!$nocheck) {
5472
5473             my $conf = PVE::QemuConfig->load_config($vmid);
5474
5475             PVE::QemuConfig->check_lock($conf)
5476                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5477         }
5478
5479         mon_cmd($vmid, $resume_cmd);
5480     });
5481 }
5482
5483 sub vm_sendkey {
5484     my ($vmid, $skiplock, $key) = @_;
5485
5486     PVE::QemuConfig->lock_config($vmid, sub {
5487
5488         my $conf = PVE::QemuConfig->load_config($vmid);
5489
5490         # there is no qmp command, so we use the human monitor command
5491         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5492         die $res if $res ne '';
5493     });
5494 }
5495
5496 # vzdump restore implementaion
5497
5498 sub tar_archive_read_firstfile {
5499     my $archive = shift;
5500
5501     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5502
5503     # try to detect archive type first
5504     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5505         die "unable to open file '$archive'\n";
5506     my $firstfile = <$fh>;
5507     kill 15, $pid;
5508     close $fh;
5509
5510     die "ERROR: archive contaions no data\n" if !$firstfile;
5511     chomp $firstfile;
5512
5513     return $firstfile;
5514 }
5515
5516 sub tar_restore_cleanup {
5517     my ($storecfg, $statfile) = @_;
5518
5519     print STDERR "starting cleanup\n";
5520
5521     if (my $fd = IO::File->new($statfile, "r")) {
5522         while (defined(my $line = <$fd>)) {
5523             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5524                 my $volid = $2;
5525                 eval {
5526                     if ($volid =~ m|^/|) {
5527                         unlink $volid || die 'unlink failed\n';
5528                     } else {
5529                         PVE::Storage::vdisk_free($storecfg, $volid);
5530                     }
5531                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5532                 };
5533                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5534             } else {
5535                 print STDERR "unable to parse line in statfile - $line";
5536             }
5537         }
5538         $fd->close();
5539     }
5540 }
5541
5542 sub restore_file_archive {
5543     my ($archive, $vmid, $user, $opts) = @_;
5544
5545     return restore_vma_archive($archive, $vmid, $user, $opts)
5546         if $archive eq '-';
5547
5548     my $info = PVE::Storage::archive_info($archive);
5549     my $format = $opts->{format} // $info->{format};
5550     my $comp = $info->{compression};
5551
5552     # try to detect archive format
5553     if ($format eq 'tar') {
5554         return restore_tar_archive($archive, $vmid, $user, $opts);
5555     } else {
5556         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5557     }
5558 }
5559
5560 # hepler to remove disks that will not be used after restore
5561 my $restore_cleanup_oldconf = sub {
5562     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
5563
5564     PVE::QemuConfig->foreach_volume($oldconf, sub {
5565         my ($ds, $drive) = @_;
5566
5567         return if drive_is_cdrom($drive, 1);
5568
5569         my $volid = $drive->{file};
5570         return if !$volid || $volid =~ m|^/|;
5571
5572         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
5573         return if !$path || !$owner || ($owner != $vmid);
5574
5575         # Note: only delete disk we want to restore
5576         # other volumes will become unused
5577         if ($virtdev_hash->{$ds}) {
5578             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
5579             if (my $err = $@) {
5580                 warn $err;
5581             }
5582         }
5583     });
5584
5585     # delete vmstate files, after the restore we have no snapshots anymore
5586     foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5587         my $snap = $oldconf->{snapshots}->{$snapname};
5588         if ($snap->{vmstate}) {
5589             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
5590             if (my $err = $@) {
5591                 warn $err;
5592             }
5593         }
5594     }
5595 };
5596
5597 # Helper to parse vzdump backup device hints
5598 #
5599 # $rpcenv: Environment, used to ckeck storage permissions
5600 # $user: User ID, to check storage permissions
5601 # $storecfg: Storage configuration
5602 # $fh: the file handle for reading the configuration
5603 # $devinfo: should contain device sizes for all backu-up'ed devices
5604 # $options: backup options (pool, default storage)
5605 #
5606 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
5607 my $parse_backup_hints = sub {
5608     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
5609
5610     my $virtdev_hash = {};
5611
5612     while (defined(my $line = <$fh>)) {
5613         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5614             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5615             die "archive does not contain data for drive '$virtdev'\n"
5616                 if !$devinfo->{$devname};
5617
5618             if (defined($options->{storage})) {
5619                 $storeid = $options->{storage} || 'local';
5620             } elsif (!$storeid) {
5621                 $storeid = 'local';
5622             }
5623             $format = 'raw' if !$format;
5624             $devinfo->{$devname}->{devname} = $devname;
5625             $devinfo->{$devname}->{virtdev} = $virtdev;
5626             $devinfo->{$devname}->{format} = $format;
5627             $devinfo->{$devname}->{storeid} = $storeid;
5628
5629             # check permission on storage
5630             my $pool = $options->{pool}; # todo: do we need that?
5631             if ($user ne 'root@pam') {
5632                 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5633             }
5634
5635             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5636         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
5637             my $virtdev = $1;
5638             my $drive = parse_drive($virtdev, $2);
5639             if (drive_is_cloudinit($drive)) {
5640                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5641                 $storeid = $options->{storage} if defined ($options->{storage});
5642                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5643                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
5644
5645                 $virtdev_hash->{$virtdev} = {
5646                     format => $format,
5647                     storeid => $storeid,
5648                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
5649                     is_cloudinit => 1,
5650                 };
5651             }
5652         }
5653     }
5654
5655     return $virtdev_hash;
5656 };
5657
5658 # Helper to allocate and activate all volumes required for a restore
5659 #
5660 # $storecfg: Storage configuration
5661 # $virtdev_hash: as returned by parse_backup_hints()
5662 #
5663 # Returns: { $virtdev => $volid }
5664 my $restore_allocate_devices = sub {
5665     my ($storecfg, $virtdev_hash, $vmid) = @_;
5666
5667     my $map = {};
5668     foreach my $virtdev (sort keys %$virtdev_hash) {
5669         my $d = $virtdev_hash->{$virtdev};
5670         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5671         my $storeid = $d->{storeid};
5672         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5673
5674         # test if requested format is supported
5675         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5676         my $supported = grep { $_ eq $d->{format} } @$validFormats;
5677         $d->{format} = $defFormat if !$supported;
5678
5679         my $name;
5680         if ($d->{is_cloudinit}) {
5681             $name = "vm-$vmid-cloudinit";
5682             $name .= ".$d->{format}" if $d->{format} ne 'raw';
5683         }
5684
5685         my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
5686
5687         print STDERR "new volume ID is '$volid'\n";
5688         $d->{volid} = $volid;
5689
5690         PVE::Storage::activate_volumes($storecfg, [$volid]);
5691
5692         $map->{$virtdev} = $volid;
5693     }
5694
5695     return $map;
5696 };
5697
5698 my $restore_update_config_line = sub {
5699     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5700
5701     return if $line =~ m/^\#qmdump\#/;
5702     return if $line =~ m/^\#vzdump\#/;
5703     return if $line =~ m/^lock:/;
5704     return if $line =~ m/^unused\d+:/;
5705     return if $line =~ m/^parent:/;
5706
5707     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5708     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5709         # try to convert old 1.X settings
5710         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5711         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5712             my ($model, $macaddr) = split(/\=/, $devconfig);
5713             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5714             my $net = {
5715                 model => $model,
5716                 bridge => "vmbr$ind",
5717                 macaddr => $macaddr,
5718             };
5719             my $netstr = print_net($net);
5720
5721             print $outfd "net$cookie->{netcount}: $netstr\n";
5722             $cookie->{netcount}++;
5723         }
5724     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5725         my ($id, $netstr) = ($1, $2);
5726         my $net = parse_net($netstr);
5727         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5728         $netstr = print_net($net);
5729         print $outfd "$id: $netstr\n";
5730     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5731         my $virtdev = $1;
5732         my $value = $3;
5733         my $di = parse_drive($virtdev, $value);
5734         if (defined($di->{backup}) && !$di->{backup}) {
5735             print $outfd "#$line";
5736         } elsif ($map->{$virtdev}) {
5737             delete $di->{format}; # format can change on restore
5738             $di->{file} = $map->{$virtdev};
5739             $value = print_drive($di);
5740             print $outfd "$virtdev: $value\n";
5741         } else {
5742             print $outfd $line;
5743         }
5744     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5745         my $vmgenid = $1;
5746         if ($vmgenid ne '0') {
5747             # always generate a new vmgenid if there was a valid one setup
5748             $vmgenid = generate_uuid();
5749         }
5750         print $outfd "vmgenid: $vmgenid\n";
5751     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5752         my ($uuid, $uuid_str);
5753         UUID::generate($uuid);
5754         UUID::unparse($uuid, $uuid_str);
5755         my $smbios1 = parse_smbios1($2);
5756         $smbios1->{uuid} = $uuid_str;
5757         print $outfd $1.print_smbios1($smbios1)."\n";
5758     } else {
5759         print $outfd $line;
5760     }
5761 };
5762
5763 my $restore_deactivate_volumes = sub {
5764     my ($storecfg, $devinfo) = @_;
5765
5766     my $vollist = [];
5767     foreach my $devname (keys %$devinfo) {
5768         my $volid = $devinfo->{$devname}->{volid};
5769         push @$vollist, $volid if $volid;
5770     }
5771
5772     PVE::Storage::deactivate_volumes($storecfg, $vollist);
5773 };
5774
5775 my $restore_destroy_volumes = sub {
5776     my ($storecfg, $devinfo) = @_;
5777
5778     foreach my $devname (keys %$devinfo) {
5779         my $volid = $devinfo->{$devname}->{volid};
5780         next if !$volid;
5781         eval {
5782             if ($volid =~ m|^/|) {
5783                 unlink $volid || die 'unlink failed\n';
5784             } else {
5785                 PVE::Storage::vdisk_free($storecfg, $volid);
5786             }
5787             print STDERR "temporary volume '$volid' sucessfuly removed\n";
5788         };
5789         print STDERR "unable to cleanup '$volid' - $@" if $@;
5790     }
5791 };
5792
5793 sub scan_volids {
5794     my ($cfg, $vmid) = @_;
5795
5796     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5797
5798     my $volid_hash = {};
5799     foreach my $storeid (keys %$info) {
5800         foreach my $item (@{$info->{$storeid}}) {
5801             next if !($item->{volid} && $item->{size});
5802             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5803             $volid_hash->{$item->{volid}} = $item;
5804         }
5805     }
5806
5807     return $volid_hash;
5808 }
5809
5810 sub update_disk_config {
5811     my ($vmid, $conf, $volid_hash) = @_;
5812
5813     my $changes;
5814     my $prefix = "VM $vmid";
5815
5816     # used and unused disks
5817     my $referenced = {};
5818
5819     # Note: it is allowed to define multiple storages with same path (alias), so
5820     # we need to check both 'volid' and real 'path' (two different volid can point
5821     # to the same path).
5822
5823     my $referencedpath = {};
5824
5825     # update size info
5826     PVE::QemuConfig->foreach_volume($conf, sub {
5827         my ($opt, $drive) = @_;
5828
5829         my $volid = $drive->{file};
5830         return if !$volid;
5831
5832         # mark volid as "in-use" for next step
5833         $referenced->{$volid} = 1;
5834         if ($volid_hash->{$volid} &&
5835             (my $path = $volid_hash->{$volid}->{path})) {
5836             $referencedpath->{$path} = 1;
5837         }
5838
5839         return if drive_is_cdrom($drive);
5840         return if !$volid_hash->{$volid};
5841
5842         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volid_hash->{$volid}->{size});
5843         if (defined($updated)) {
5844             $changes = 1;
5845             $conf->{$opt} = print_drive($updated);
5846             print "$prefix ($opt): $msg\n";
5847         }
5848     });
5849
5850     # remove 'unusedX' entry if volume is used
5851     PVE::QemuConfig->foreach_unused_volume($conf, sub {
5852         my ($opt, $drive) = @_;
5853
5854         my $volid = $drive->{file};
5855         return if !$volid;
5856
5857         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5858         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5859             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
5860             $changes = 1;
5861             delete $conf->{$opt};
5862         }
5863
5864         $referenced->{$volid} = 1;
5865         $referencedpath->{$path} = 1 if $path;
5866     });
5867
5868     foreach my $volid (sort keys %$volid_hash) {
5869         next if $volid =~ m/vm-$vmid-state-/;
5870         next if $referenced->{$volid};
5871         my $path = $volid_hash->{$volid}->{path};
5872         next if !$path; # just to be sure
5873         next if $referencedpath->{$path};
5874         $changes = 1;
5875         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
5876         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
5877         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5878     }
5879
5880     return $changes;
5881 }
5882
5883 sub rescan {
5884     my ($vmid, $nolock, $dryrun) = @_;
5885
5886     my $cfg = PVE::Storage::config();
5887
5888     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
5889     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
5890     foreach my $stor (keys %{$cfg->{ids}}) {
5891         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
5892     }
5893
5894     print "rescan volumes...\n";
5895     my $volid_hash = scan_volids($cfg, $vmid);
5896
5897     my $updatefn =  sub {
5898         my ($vmid) = @_;
5899
5900         my $conf = PVE::QemuConfig->load_config($vmid);
5901
5902         PVE::QemuConfig->check_lock($conf);
5903
5904         my $vm_volids = {};
5905         foreach my $volid (keys %$volid_hash) {
5906             my $info = $volid_hash->{$volid};
5907             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5908         }
5909
5910         my $changes = update_disk_config($vmid, $conf, $vm_volids);
5911
5912         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
5913     };
5914
5915     if (defined($vmid)) {
5916         if ($nolock) {
5917             &$updatefn($vmid);
5918         } else {
5919             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5920         }
5921     } else {
5922         my $vmlist = config_list();
5923         foreach my $vmid (keys %$vmlist) {
5924             if ($nolock) {
5925                 &$updatefn($vmid);
5926             } else {
5927                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5928             }
5929         }
5930     }
5931 }
5932
5933 sub restore_proxmox_backup_archive {
5934     my ($archive, $vmid, $user, $options) = @_;
5935
5936     my $storecfg = PVE::Storage::config();
5937
5938     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
5939     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5940
5941     my $server = $scfg->{server};
5942     my $datastore = $scfg->{datastore};
5943     my $username = $scfg->{username} // 'root@pam';
5944     my $fingerprint = $scfg->{fingerprint};
5945     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
5946
5947     my $repo = "$username\@$server:$datastore";
5948
5949     # This is only used for `pbs-restore`!
5950     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
5951     local $ENV{PBS_PASSWORD} = $password;
5952     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
5953
5954     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
5955         PVE::Storage::parse_volname($storecfg, $archive);
5956
5957     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
5958
5959     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
5960
5961     my $tmpdir = "/var/tmp/vzdumptmp$$";
5962     rmtree $tmpdir;
5963     mkpath $tmpdir;
5964
5965     my $conffile = PVE::QemuConfig->config_file($vmid);
5966     my $tmpfn = "$conffile.$$.tmp";
5967      # disable interrupts (always do cleanups)
5968     local $SIG{INT} =
5969         local $SIG{TERM} =
5970         local $SIG{QUIT} =
5971         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
5972
5973     # Note: $oldconf is undef if VM does not exists
5974     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5975     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5976
5977     my $rpcenv = PVE::RPCEnvironment::get();
5978     my $devinfo = {};
5979
5980     eval {
5981         # enable interrupts
5982         local $SIG{INT} =
5983             local $SIG{TERM} =
5984             local $SIG{QUIT} =
5985             local $SIG{HUP} =
5986             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5987
5988         my $cfgfn = "$tmpdir/qemu-server.conf";
5989         my $firewall_config_fn = "$tmpdir/fw.conf";
5990         my $index_fn = "$tmpdir/index.json";
5991
5992         my $cmd = "restore";
5993
5994         my $param = [$pbs_backup_name, "index.json", $index_fn];
5995         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
5996         my $index = PVE::Tools::file_get_contents($index_fn);
5997         $index = decode_json($index);
5998
5999         # print Dumper($index);
6000         foreach my $info (@{$index->{files}}) {
6001             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6002                 my $devname = $1;
6003                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6004                     $devinfo->{$devname}->{size} = $1;
6005                 } else {
6006                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6007                 }
6008             }
6009         }
6010
6011         my $is_qemu_server_backup = scalar(grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}});
6012         if (!$is_qemu_server_backup) {
6013             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6014         }
6015         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6016
6017         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6018         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6019
6020         if ($has_firewall_config) {
6021             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6022             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6023
6024             my $pve_firewall_dir = '/etc/pve/firewall';
6025             mkdir $pve_firewall_dir; # make sure the dir exists
6026             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6027         }
6028
6029         my $fh = IO::File->new($cfgfn, "r") ||
6030             die "unable to read qemu-server.conf - $!\n";
6031
6032         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6033
6034         # fixme: rate limit?
6035
6036         # create empty/temp config
6037         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6038
6039         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6040
6041         # allocate volumes
6042         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6043
6044         foreach my $virtdev (sort keys %$virtdev_hash) {
6045             my $d = $virtdev_hash->{$virtdev};
6046             next if $d->{is_cloudinit}; # no need to restore cloudinit
6047
6048             my $volid = $d->{volid};
6049
6050             my $path = PVE::Storage::path($storecfg, $volid);
6051
6052             # This is the ONLY user of the PBS_ env vars set on top of this function!
6053             my $pbs_restore_cmd = [
6054                 '/usr/bin/pbs-restore',
6055                 '--repository', $repo,
6056                 $pbs_backup_name,
6057                 "$d->{devname}.img.fidx",
6058                 $path,
6059                 '--verbose',
6060                 ];
6061
6062             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6063             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
6064
6065             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6066                 push @$pbs_restore_cmd, '--skip-zero';
6067             }
6068
6069             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6070             print "restore proxmox backup image: $dbg_cmdstring\n";
6071             run_command($pbs_restore_cmd);
6072         }
6073
6074         $fh->seek(0, 0) || die "seek failed - $!\n";
6075
6076         my $outfd = new IO::File ($tmpfn, "w") ||
6077             die "unable to write config for VM $vmid\n";
6078
6079         my $cookie = { netcount => 0 };
6080         while (defined(my $line = <$fh>)) {
6081             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $options->{unique});
6082         }
6083
6084         $fh->close();
6085         $outfd->close();
6086     };
6087     my $err = $@;
6088
6089     $restore_deactivate_volumes->($storecfg, $devinfo);
6090
6091     rmtree $tmpdir;
6092
6093     if ($err) {
6094         unlink $tmpfn;
6095         $restore_destroy_volumes->($storecfg, $devinfo);
6096         die $err;
6097     }
6098
6099     rename($tmpfn, $conffile) ||
6100         die "unable to commit configuration file '$conffile'\n";
6101
6102     PVE::Cluster::cfs_update(); # make sure we read new file
6103
6104     eval { rescan($vmid, 1); };
6105     warn $@ if $@;
6106 }
6107
6108 sub restore_vma_archive {
6109     my ($archive, $vmid, $user, $opts, $comp) = @_;
6110
6111     my $readfrom = $archive;
6112
6113     my $cfg = PVE::Storage::config();
6114     my $commands = [];
6115     my $bwlimit = $opts->{bwlimit};
6116
6117     my $dbg_cmdstring = '';
6118     my $add_pipe = sub {
6119         my ($cmd) = @_;
6120         push @$commands, $cmd;
6121         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6122         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6123         $readfrom = '-';
6124     };
6125
6126     my $input = undef;
6127     if ($archive eq '-') {
6128         $input = '<&STDIN';
6129     } else {
6130         # If we use a backup from a PVE defined storage we also consider that
6131         # storage's rate limit:
6132         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6133         if (defined($volid)) {
6134             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6135             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6136             if ($readlimit) {
6137                 print STDERR "applying read rate limit: $readlimit\n";
6138                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6139                 $add_pipe->($cstream);
6140             }
6141         }
6142     }
6143
6144     if ($comp) {
6145         my $info = PVE::Storage::decompressor_info('vma', $comp);
6146         my $cmd = $info->{decompressor};
6147         push @$cmd, $readfrom;
6148         $add_pipe->($cmd);
6149     }
6150
6151     my $tmpdir = "/var/tmp/vzdumptmp$$";
6152     rmtree $tmpdir;
6153
6154     # disable interrupts (always do cleanups)
6155     local $SIG{INT} =
6156         local $SIG{TERM} =
6157         local $SIG{QUIT} =
6158         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6159
6160     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6161     POSIX::mkfifo($mapfifo, 0600);
6162     my $fifofh;
6163
6164     my $openfifo = sub {
6165         open($fifofh, '>', $mapfifo) || die $!;
6166     };
6167
6168     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6169
6170     my $oldtimeout;
6171     my $timeout = 5;
6172
6173     my $devinfo = {};
6174
6175     my $rpcenv = PVE::RPCEnvironment::get();
6176
6177     my $conffile = PVE::QemuConfig->config_file($vmid);
6178     my $tmpfn = "$conffile.$$.tmp";
6179
6180     # Note: $oldconf is undef if VM does not exist
6181     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6182     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6183
6184     my %storage_limits;
6185
6186     my $print_devmap = sub {
6187         my $cfgfn = "$tmpdir/qemu-server.conf";
6188
6189         # we can read the config - that is already extracted
6190         my $fh = IO::File->new($cfgfn, "r") ||
6191             die "unable to read qemu-server.conf - $!\n";
6192
6193         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6194         if (-f $fwcfgfn) {
6195             my $pve_firewall_dir = '/etc/pve/firewall';
6196             mkdir $pve_firewall_dir; # make sure the dir exists
6197             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6198         }
6199
6200         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6201
6202         foreach my $key (keys %storage_limits) {
6203             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6204             next if !$limit;
6205             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6206             $storage_limits{$key} = $limit * 1024;
6207         }
6208
6209         foreach my $devname (keys %$devinfo) {
6210             die "found no device mapping information for device '$devname'\n"
6211                 if !$devinfo->{$devname}->{virtdev};
6212         }
6213
6214         # create empty/temp config
6215         if ($oldconf) {
6216             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6217             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6218         }
6219
6220         # allocate volumes
6221         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6222
6223         # print restore information to $fifofh
6224         foreach my $virtdev (sort keys %$virtdev_hash) {
6225             my $d = $virtdev_hash->{$virtdev};
6226             next if $d->{is_cloudinit}; # no need to restore cloudinit
6227
6228             my $storeid = $d->{storeid};
6229             my $volid = $d->{volid};
6230
6231             my $map_opts = '';
6232             if (my $limit = $storage_limits{$storeid}) {
6233                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6234             }
6235
6236             my $write_zeros = 1;
6237             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6238                 $write_zeros = 0;
6239             }
6240
6241             my $path = PVE::Storage::path($cfg, $volid);
6242
6243             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6244
6245             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6246         }
6247
6248         $fh->seek(0, 0) || die "seek failed - $!\n";
6249
6250         my $outfd = new IO::File ($tmpfn, "w") ||
6251             die "unable to write config for VM $vmid\n";
6252
6253         my $cookie = { netcount => 0 };
6254         while (defined(my $line = <$fh>)) {
6255             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6256         }
6257
6258         $fh->close();
6259         $outfd->close();
6260     };
6261
6262     eval {
6263         # enable interrupts
6264         local $SIG{INT} =
6265             local $SIG{TERM} =
6266             local $SIG{QUIT} =
6267             local $SIG{HUP} =
6268             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6269         local $SIG{ALRM} = sub { die "got timeout\n"; };
6270
6271         $oldtimeout = alarm($timeout);
6272
6273         my $parser = sub {
6274             my $line = shift;
6275
6276             print "$line\n";
6277
6278             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6279                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6280                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6281             } elsif ($line =~ m/^CTIME: /) {
6282                 # we correctly received the vma config, so we can disable
6283                 # the timeout now for disk allocation (set to 10 minutes, so
6284                 # that we always timeout if something goes wrong)
6285                 alarm(600);
6286                 &$print_devmap();
6287                 print $fifofh "done\n";
6288                 my $tmp = $oldtimeout || 0;
6289                 $oldtimeout = undef;
6290                 alarm($tmp);
6291                 close($fifofh);
6292             }
6293         };
6294
6295         print "restore vma archive: $dbg_cmdstring\n";
6296         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6297     };
6298     my $err = $@;
6299
6300     alarm($oldtimeout) if $oldtimeout;
6301
6302     $restore_deactivate_volumes->($cfg, $devinfo);
6303
6304     unlink $mapfifo;
6305     rmtree $tmpdir;
6306
6307     if ($err) {
6308         unlink $tmpfn;
6309         $restore_destroy_volumes->($cfg, $devinfo);
6310         die $err;
6311     }
6312
6313     rename($tmpfn, $conffile) ||
6314         die "unable to commit configuration file '$conffile'\n";
6315
6316     PVE::Cluster::cfs_update(); # make sure we read new file
6317
6318     eval { rescan($vmid, 1); };
6319     warn $@ if $@;
6320 }
6321
6322 sub restore_tar_archive {
6323     my ($archive, $vmid, $user, $opts) = @_;
6324
6325     if ($archive ne '-') {
6326         my $firstfile = tar_archive_read_firstfile($archive);
6327         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6328             if $firstfile ne 'qemu-server.conf';
6329     }
6330
6331     my $storecfg = PVE::Storage::config();
6332
6333     # avoid zombie disks when restoring over an existing VM -> cleanup first
6334     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6335     # skiplock=1 because qmrestore has set the 'create' lock itself already
6336     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6337     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6338
6339     my $tocmd = "/usr/lib/qemu-server/qmextract";
6340
6341     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6342     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6343     $tocmd .= ' --prealloc' if $opts->{prealloc};
6344     $tocmd .= ' --info' if $opts->{info};
6345
6346     # tar option "xf" does not autodetect compression when read from STDIN,
6347     # so we pipe to zcat
6348     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6349         PVE::Tools::shellquote("--to-command=$tocmd");
6350
6351     my $tmpdir = "/var/tmp/vzdumptmp$$";
6352     mkpath $tmpdir;
6353
6354     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6355     local $ENV{VZDUMP_VMID} = $vmid;
6356     local $ENV{VZDUMP_USER} = $user;
6357
6358     my $conffile = PVE::QemuConfig->config_file($vmid);
6359     my $tmpfn = "$conffile.$$.tmp";
6360
6361     # disable interrupts (always do cleanups)
6362     local $SIG{INT} =
6363         local $SIG{TERM} =
6364         local $SIG{QUIT} =
6365         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6366
6367     eval {
6368         # enable interrupts
6369         local $SIG{INT} =
6370             local $SIG{TERM} =
6371             local $SIG{QUIT} =
6372             local $SIG{HUP} =
6373             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6374
6375         if ($archive eq '-') {
6376             print "extracting archive from STDIN\n";
6377             run_command($cmd, input => "<&STDIN");
6378         } else {
6379             print "extracting archive '$archive'\n";
6380             run_command($cmd);
6381         }
6382
6383         return if $opts->{info};
6384
6385         # read new mapping
6386         my $map = {};
6387         my $statfile = "$tmpdir/qmrestore.stat";
6388         if (my $fd = IO::File->new($statfile, "r")) {
6389             while (defined (my $line = <$fd>)) {
6390                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6391                     $map->{$1} = $2 if $1;
6392                 } else {
6393                     print STDERR "unable to parse line in statfile - $line\n";
6394                 }
6395             }
6396             $fd->close();
6397         }
6398
6399         my $confsrc = "$tmpdir/qemu-server.conf";
6400
6401         my $srcfd = new IO::File($confsrc, "r") ||
6402             die "unable to open file '$confsrc'\n";
6403
6404         my $outfd = new IO::File ($tmpfn, "w") ||
6405             die "unable to write config for VM $vmid\n";
6406
6407         my $cookie = { netcount => 0 };
6408         while (defined (my $line = <$srcfd>)) {
6409             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6410         }
6411
6412         $srcfd->close();
6413         $outfd->close();
6414     };
6415     if (my $err = $@) {
6416         unlink $tmpfn;
6417         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6418         die $err;
6419     }
6420
6421     rmtree $tmpdir;
6422
6423     rename $tmpfn, $conffile ||
6424         die "unable to commit configuration file '$conffile'\n";
6425
6426     PVE::Cluster::cfs_update(); # make sure we read new file
6427
6428     eval { rescan($vmid, 1); };
6429     warn $@ if $@;
6430 };
6431
6432 sub foreach_storage_used_by_vm {
6433     my ($conf, $func) = @_;
6434
6435     my $sidhash = {};
6436
6437     PVE::QemuConfig->foreach_volume($conf, sub {
6438         my ($ds, $drive) = @_;
6439         return if drive_is_cdrom($drive);
6440
6441         my $volid = $drive->{file};
6442
6443         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6444         $sidhash->{$sid} = $sid if $sid;
6445     });
6446
6447     foreach my $sid (sort keys %$sidhash) {
6448         &$func($sid);
6449     }
6450 }
6451
6452 my $qemu_snap_storage = {
6453     rbd => 1,
6454 };
6455 sub do_snapshots_with_qemu {
6456     my ($storecfg, $volid) = @_;
6457
6458     my $storage_name = PVE::Storage::parse_volume_id($volid);
6459     my $scfg = $storecfg->{ids}->{$storage_name};
6460
6461     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6462         return 1;
6463     }
6464
6465     if ($volid =~ m/\.(qcow2|qed)$/){
6466         return 1;
6467     }
6468
6469     return undef;
6470 }
6471
6472 sub qga_check_running {
6473     my ($vmid, $nowarn) = @_;
6474
6475     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6476     if ($@) {
6477         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6478         return 0;
6479     }
6480     return 1;
6481 }
6482
6483 sub template_create {
6484     my ($vmid, $conf, $disk) = @_;
6485
6486     my $storecfg = PVE::Storage::config();
6487
6488     PVE::QemuConfig->foreach_volume($conf, sub {
6489         my ($ds, $drive) = @_;
6490
6491         return if drive_is_cdrom($drive);
6492         return if $disk && $ds ne $disk;
6493
6494         my $volid = $drive->{file};
6495         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6496
6497         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6498         $drive->{file} = $voliddst;
6499         $conf->{$ds} = print_drive($drive);
6500         PVE::QemuConfig->write_config($vmid, $conf);
6501     });
6502 }
6503
6504 sub convert_iscsi_path {
6505     my ($path) = @_;
6506
6507     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6508         my $portal = $1;
6509         my $target = $2;
6510         my $lun = $3;
6511
6512         my $initiator_name = get_initiator_name();
6513
6514         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6515                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6516     }
6517
6518     die "cannot convert iscsi path '$path', unkown format\n";
6519 }
6520
6521 sub qemu_img_convert {
6522     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6523
6524     my $storecfg = PVE::Storage::config();
6525     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6526     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6527
6528     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6529
6530     my $cachemode;
6531     my $src_path;
6532     my $src_is_iscsi = 0;
6533     my $src_format;
6534
6535     if ($src_storeid) {
6536         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6537         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6538         $src_format = qemu_img_format($src_scfg, $src_volname);
6539         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6540         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6541         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6542     } elsif (-f $src_volid) {
6543         $src_path = $src_volid;
6544         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6545             $src_format = $1;
6546         }
6547     }
6548
6549     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6550
6551     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6552     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6553     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6554     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6555
6556     my $cmd = [];
6557     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6558     push @$cmd, '-l', "snapshot.name=$snapname"
6559         if $snapname && $src_format && $src_format eq "qcow2";
6560     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6561     push @$cmd, '-T', $cachemode if defined($cachemode);
6562
6563     if ($src_is_iscsi) {
6564         push @$cmd, '--image-opts';
6565         $src_path = convert_iscsi_path($src_path);
6566     } elsif ($src_format) {
6567         push @$cmd, '-f', $src_format;
6568     }
6569
6570     if ($dst_is_iscsi) {
6571         push @$cmd, '--target-image-opts';
6572         $dst_path = convert_iscsi_path($dst_path);
6573     } else {
6574         push @$cmd, '-O', $dst_format;
6575     }
6576
6577     push @$cmd, $src_path;
6578
6579     if (!$dst_is_iscsi && $is_zero_initialized) {
6580         push @$cmd, "zeroinit:$dst_path";
6581     } else {
6582         push @$cmd, $dst_path;
6583     }
6584
6585     my $parser = sub {
6586         my $line = shift;
6587         if($line =~ m/\((\S+)\/100\%\)/){
6588             my $percent = $1;
6589             my $transferred = int($size * $percent / 100);
6590             my $remaining = $size - $transferred;
6591
6592             print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6593         }
6594
6595     };
6596
6597     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6598     my $err = $@;
6599     die "copy failed: $err" if $err;
6600 }
6601
6602 sub qemu_img_format {
6603     my ($scfg, $volname) = @_;
6604
6605     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6606         return $1;
6607     } else {
6608         return "raw";
6609     }
6610 }
6611
6612 sub qemu_drive_mirror {
6613     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
6614
6615     $jobs = {} if !$jobs;
6616
6617     my $qemu_target;
6618     my $format;
6619     $jobs->{"drive-$drive"} = {};
6620
6621     if ($dst_volid =~ /^nbd:/) {
6622         $qemu_target = $dst_volid;
6623         $format = "nbd";
6624     } else {
6625         my $storecfg = PVE::Storage::config();
6626         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6627
6628         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6629
6630         $format = qemu_img_format($dst_scfg, $dst_volname);
6631
6632         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6633
6634         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6635     }
6636
6637     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6638     $opts->{format} = $format if $format;
6639
6640     if (defined($src_bitmap)) {
6641         $opts->{sync} = 'incremental';
6642         $opts->{bitmap} = $src_bitmap;
6643         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
6644     }
6645
6646     if (defined($bwlimit)) {
6647         $opts->{speed} = $bwlimit * 1024;
6648         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6649     } else {
6650         print "drive mirror is starting for drive-$drive\n";
6651     }
6652
6653     # if a job already runs for this device we get an error, catch it for cleanup
6654     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
6655     if (my $err = $@) {
6656         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6657         warn "$@\n" if $@;
6658         die "mirroring error: $err\n";
6659     }
6660
6661     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
6662 }
6663
6664 # $completion can be either
6665 # 'complete': wait until all jobs are ready, block-job-complete them (default)
6666 # 'cancel': wait until all jobs are ready, block-job-cancel them
6667 # 'skip': wait until all jobs are ready, return with block jobs in ready state
6668 sub qemu_drive_mirror_monitor {
6669     my ($vmid, $vmiddst, $jobs, $completion, $qga) = @_;
6670
6671     $completion //= 'complete';
6672
6673     eval {
6674         my $err_complete = 0;
6675
6676         while (1) {
6677             die "storage migration timed out\n" if $err_complete > 300;
6678
6679             my $stats = mon_cmd($vmid, "query-block-jobs");
6680
6681             my $running_mirror_jobs = {};
6682             foreach my $stat (@$stats) {
6683                 next if $stat->{type} ne 'mirror';
6684                 $running_mirror_jobs->{$stat->{device}} = $stat;
6685             }
6686
6687             my $readycounter = 0;
6688
6689             foreach my $job (keys %$jobs) {
6690
6691                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6692                     print "$job : finished\n";
6693                     delete $jobs->{$job};
6694                     next;
6695                 }
6696
6697                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6698
6699                 my $busy = $running_mirror_jobs->{$job}->{busy};
6700                 my $ready = $running_mirror_jobs->{$job}->{ready};
6701                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6702                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6703                     my $remaining = $total - $transferred;
6704                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6705
6706                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6707                 }
6708
6709                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6710             }
6711
6712             last if scalar(keys %$jobs) == 0;
6713
6714             if ($readycounter == scalar(keys %$jobs)) {
6715                 print "all mirroring jobs are ready \n";
6716                 last if $completion eq 'skip'; #do the complete later
6717
6718                 if ($vmiddst && $vmiddst != $vmid) {
6719                     my $agent_running = $qga && qga_check_running($vmid);
6720                     if ($agent_running) {
6721                         print "freeze filesystem\n";
6722                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6723                     } else {
6724                         print "suspend vm\n";
6725                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6726                     }
6727
6728                     # if we clone a disk for a new target vm, we don't switch the disk
6729                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6730
6731                     if ($agent_running) {
6732                         print "unfreeze filesystem\n";
6733                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6734                     } else {
6735                         print "resume vm\n";
6736                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6737                     }
6738
6739                     last;
6740                 } else {
6741
6742                     foreach my $job (keys %$jobs) {
6743                         # try to switch the disk if source and destination are on the same guest
6744                         print "$job: Completing block job...\n";
6745
6746                         my $op;
6747                         if ($completion eq 'complete') {
6748                             $op = 'block-job-complete';
6749                         } elsif ($completion eq 'cancel') {
6750                             $op = 'block-job-cancel';
6751                         } else {
6752                             die "invalid completion value: $completion\n";
6753                         }
6754                         eval { mon_cmd($vmid, $op, device => $job) };
6755                         if ($@ =~ m/cannot be completed/) {
6756                             print "$job: Block job cannot be completed, try again.\n";
6757                             $err_complete++;
6758                         }else {
6759                             print "$job: Completed successfully.\n";
6760                             $jobs->{$job}->{complete} = 1;
6761                         }
6762                     }
6763                 }
6764             }
6765             sleep 1;
6766         }
6767     };
6768     my $err = $@;
6769
6770     if ($err) {
6771         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6772         die "mirroring error: $err";
6773     }
6774
6775 }
6776
6777 sub qemu_blockjobs_cancel {
6778     my ($vmid, $jobs) = @_;
6779
6780     foreach my $job (keys %$jobs) {
6781         print "$job: Cancelling block job\n";
6782         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
6783         $jobs->{$job}->{cancel} = 1;
6784     }
6785
6786     while (1) {
6787         my $stats = mon_cmd($vmid, "query-block-jobs");
6788
6789         my $running_jobs = {};
6790         foreach my $stat (@$stats) {
6791             $running_jobs->{$stat->{device}} = $stat;
6792         }
6793
6794         foreach my $job (keys %$jobs) {
6795
6796             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6797                 print "$job: Done.\n";
6798                 delete $jobs->{$job};
6799             }
6800         }
6801
6802         last if scalar(keys %$jobs) == 0;
6803
6804         sleep 1;
6805     }
6806 }
6807
6808 sub clone_disk {
6809     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6810         $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
6811
6812     my $newvolid;
6813
6814     if (!$full) {
6815         print "create linked clone of drive $drivename ($drive->{file})\n";
6816         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6817         push @$newvollist, $newvolid;
6818     } else {
6819
6820         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6821         $storeid = $storage if $storage;
6822
6823         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6824         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6825
6826         print "create full clone of drive $drivename ($drive->{file})\n";
6827         my $name = undef;
6828         if (drive_is_cloudinit($drive)) {
6829             $name = "vm-$newvmid-cloudinit";
6830             $name .= ".$dst_format" if $dst_format ne 'raw';
6831             $snapname = undef;
6832             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
6833         } elsif ($drivename eq 'efidisk0') {
6834             $size = get_efivars_size($conf);
6835         }
6836         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6837         push @$newvollist, $newvolid;
6838
6839         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6840
6841         if (drive_is_cloudinit($drive)) {
6842             goto no_data_clone;
6843         }
6844
6845         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6846         if (!$running || $snapname) {
6847             # TODO: handle bwlimits
6848             if ($drivename eq 'efidisk0') {
6849                 # the relevant data on the efidisk may be smaller than the source
6850                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
6851                 # that is given by the OVMF_VARS.fd
6852                 my $src_path = PVE::Storage::path($storecfg, $drive->{file});
6853                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
6854                 run_command(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=1", "count=$size", "if=$src_path", "of=$dst_path"]);
6855             } else {
6856                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6857             }
6858         } else {
6859
6860             my $kvmver = get_running_qemu_version ($vmid);
6861             if (!min_version($kvmver, 2, 7)) {
6862                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6863                     if $drive->{iothread};
6864             }
6865
6866             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $completion, $qga, $bwlimit);
6867         }
6868     }
6869
6870 no_data_clone:
6871     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6872
6873     my $disk = $drive;
6874     $disk->{format} = undef;
6875     $disk->{file} = $newvolid;
6876     $disk->{size} = $size;
6877
6878     return $disk;
6879 }
6880
6881 sub get_running_qemu_version {
6882     my ($vmid) = @_;
6883     my $res = mon_cmd($vmid, "query-version");
6884     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6885 }
6886
6887 sub qemu_use_old_bios_files {
6888     my ($machine_type) = @_;
6889
6890     return if !$machine_type;
6891
6892     my $use_old_bios_files = undef;
6893
6894     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6895         $machine_type = $1;
6896         $use_old_bios_files = 1;
6897     } else {
6898         my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
6899         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6900         # load new efi bios files on migration. So this hack is required to allow
6901         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6902         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6903         $use_old_bios_files = !min_version($version, 2, 4);
6904     }
6905
6906     return ($use_old_bios_files, $machine_type);
6907 }
6908
6909 sub get_efivars_size {
6910     my ($conf) = @_;
6911     my $arch = get_vm_arch($conf);
6912     my (undef, $ovmf_vars) = get_ovmf_files($arch);
6913     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
6914     return -s $ovmf_vars;
6915 }
6916
6917 sub update_efidisk_size {
6918     my ($conf) = @_;
6919
6920     return if !defined($conf->{efidisk0});
6921
6922     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
6923     $disk->{size} = get_efivars_size($conf);
6924     $conf->{efidisk0} = print_drive($disk);
6925
6926     return;
6927 }
6928
6929 sub create_efidisk($$$$$) {
6930     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
6931
6932     my (undef, $ovmf_vars) = get_ovmf_files($arch);
6933     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
6934
6935     my $vars_size_b = -s $ovmf_vars;
6936     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
6937     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6938     PVE::Storage::activate_volumes($storecfg, [$volid]);
6939
6940     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
6941     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
6942
6943     return ($volid, $size/1024);
6944 }
6945
6946 sub vm_iothreads_list {
6947     my ($vmid) = @_;
6948
6949     my $res = mon_cmd($vmid, 'query-iothreads');
6950
6951     my $iothreads = {};
6952     foreach my $iothread (@$res) {
6953         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6954     }
6955
6956     return $iothreads;
6957 }
6958
6959 sub scsihw_infos {
6960     my ($conf, $drive) = @_;
6961
6962     my $maxdev = 0;
6963
6964     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6965         $maxdev = 7;
6966     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6967         $maxdev = 1;
6968     } else {
6969         $maxdev = 256;
6970     }
6971
6972     my $controller = int($drive->{index} / $maxdev);
6973     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6974
6975     return ($maxdev, $controller, $controller_prefix);
6976 }
6977
6978 sub windows_version {
6979     my ($ostype) = @_;
6980
6981     return 0 if !$ostype;
6982
6983     my $winversion = 0;
6984
6985     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6986         $winversion = 5;
6987     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6988         $winversion = 6;
6989     } elsif ($ostype =~ m/^win(\d+)$/) {
6990         $winversion = $1;
6991     }
6992
6993     return $winversion;
6994 }
6995
6996 sub resolve_dst_disk_format {
6997         my ($storecfg, $storeid, $src_volname, $format) = @_;
6998         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6999
7000         if (!$format) {
7001             # if no target format is specified, use the source disk format as hint
7002             if ($src_volname) {
7003                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7004                 $format = qemu_img_format($scfg, $src_volname);
7005             } else {
7006                 return $defFormat;
7007             }
7008         }
7009
7010         # test if requested format is supported - else use default
7011         my $supported = grep { $_ eq $format } @$validFormats;
7012         $format = $defFormat if !$supported;
7013         return $format;
7014 }
7015
7016 # NOTE: if this logic changes, please update docs & possibly gui logic
7017 sub find_vmstate_storage {
7018     my ($conf, $storecfg) = @_;
7019
7020     # first, return storage from conf if set
7021     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7022
7023     my ($target, $shared, $local);
7024
7025     foreach_storage_used_by_vm($conf, sub {
7026         my ($sid) = @_;
7027         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7028         my $dst = $scfg->{shared} ? \$shared : \$local;
7029         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7030     });
7031
7032     # second, use shared storage where VM has at least one disk
7033     # third, use local storage where VM has at least one disk
7034     # fall back to local storage
7035     $target = $shared // $local // 'local';
7036
7037     return $target;
7038 }
7039
7040 sub generate_uuid {
7041     my ($uuid, $uuid_str);
7042     UUID::generate($uuid);
7043     UUID::unparse($uuid, $uuid_str);
7044     return $uuid_str;
7045 }
7046
7047 sub generate_smbios1_uuid {
7048     return "uuid=".generate_uuid();
7049 }
7050
7051 sub nbd_stop {
7052     my ($vmid) = @_;
7053
7054     mon_cmd($vmid, 'nbd-server-stop');
7055 }
7056
7057 sub create_reboot_request {
7058     my ($vmid) = @_;
7059     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7060         or die "failed to create reboot trigger file: $!\n";
7061     close($fh);
7062 }
7063
7064 sub clear_reboot_request {
7065     my ($vmid) = @_;
7066     my $path = "/run/qemu-server/$vmid.reboot";
7067     my $res = 0;
7068
7069     $res = unlink($path);
7070     die "could not remove reboot request for $vmid: $!"
7071         if !$res && $! != POSIX::ENOENT;
7072
7073     return $res;
7074 }
7075
7076 # bash completion helper
7077
7078 sub complete_backup_archives {
7079     my ($cmdname, $pname, $cvalue) = @_;
7080
7081     my $cfg = PVE::Storage::config();
7082
7083     my $storeid;
7084
7085     if ($cvalue =~ m/^([^:]+):/) {
7086         $storeid = $1;
7087     }
7088
7089     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7090
7091     my $res = [];
7092     foreach my $id (keys %$data) {
7093         foreach my $item (@{$data->{$id}}) {
7094             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
7095             push @$res, $item->{volid} if defined($item->{volid});
7096         }
7097     }
7098
7099     return $res;
7100 }
7101
7102 my $complete_vmid_full = sub {
7103     my ($running) = @_;
7104
7105     my $idlist = vmstatus();
7106
7107     my $res = [];
7108
7109     foreach my $id (keys %$idlist) {
7110         my $d = $idlist->{$id};
7111         if (defined($running)) {
7112             next if $d->{template};
7113             next if $running && $d->{status} ne 'running';
7114             next if !$running && $d->{status} eq 'running';
7115         }
7116         push @$res, $id;
7117
7118     }
7119     return $res;
7120 };
7121
7122 sub complete_vmid {
7123     return &$complete_vmid_full();
7124 }
7125
7126 sub complete_vmid_stopped {
7127     return &$complete_vmid_full(0);
7128 }
7129
7130 sub complete_vmid_running {
7131     return &$complete_vmid_full(1);
7132 }
7133
7134 sub complete_storage {
7135
7136     my $cfg = PVE::Storage::config();
7137     my $ids = $cfg->{ids};
7138
7139     my $res = [];
7140     foreach my $sid (keys %$ids) {
7141         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7142         next if !$ids->{$sid}->{content}->{images};
7143         push @$res, $sid;
7144     }
7145
7146     return $res;
7147 }
7148
7149 sub complete_migration_storage {
7150     my ($cmd, $param, $current_value, $all_args) = @_;
7151
7152     my $targetnode = @$all_args[1];
7153
7154     my $cfg = PVE::Storage::config();
7155     my $ids = $cfg->{ids};
7156
7157     my $res = [];
7158     foreach my $sid (keys %$ids) {
7159         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7160         next if !$ids->{$sid}->{content}->{images};
7161         push @$res, $sid;
7162     }
7163
7164     return $res;
7165 }
7166
7167 1;