PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::DataCenterConfig;
  31 use PVE::Exception qw(raise raise_param_exc);
  32 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  33 use PVE::INotify;
  34 use PVE::JSONSchema qw(get_standard_option);
  35 use PVE::ProcFSTools;
  36 use PVE::RPCEnvironment;
  37 use PVE::Storage;
  38 use PVE::SysFSTools;
  39 use PVE::Systemd;
  40 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  41
  42 use PVE::QMPClient;
  43 use PVE::QemuConfig;
  44 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  45 use PVE::QemuServer::Cloudinit;
  46 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  47 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom parse_drive print_drive);
  48 use PVE::QemuServer::Machine;
  49 use PVE::QemuServer::Memory;
  50 use PVE::QemuServer::Monitor qw(mon_cmd);
  51 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  52 use PVE::QemuServer::USB qw(parse_usb_device);
  53
  54 my $have_sdn;
  55 eval {
  56     require PVE::Network::SDN::Zones;
  57     $have_sdn = 1;
  58 };
  59
  60 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  61 my $OVMF = {
  62     x86_64 => [
  63         "$EDK2_FW_BASE/OVMF_CODE.fd",
  64         "$EDK2_FW_BASE/OVMF_VARS.fd"
  65     ],
  66     aarch64 => [
  67         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  68         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  69     ],
  70 };
  71
  72 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  73
  74 # Note about locking: we use flock on the config file protect
  75 # against concurent actions.
  76 # Aditionaly, we have a 'lock' setting in the config file. This
  77 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  78 # allowed when such lock is set. But you can ignore this kind of
  79 # lock with the --skiplock flag.
  80
  81 cfs_register_file('/qemu-server/',
  82                   \&parse_vm_config,
  83                   \&write_vm_config);
  84
  85 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  86     description => "Some command save/restore state from this location.",
  87     type => 'string',
  88     maxLength => 128,
  89     optional => 1,
  90 });
  91
  92 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  93         description => "Specifies the Qemu machine type.",
  94         type => 'string',
  95         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
  96         maxLength => 40,
  97         optional => 1,
  98 });
  99
 100
 101 sub map_storage {
 102     my ($map, $source) = @_;
 103
 104     return $source if !defined($map);
 105
 106     return $map->{entries}->{$source}
 107         if $map->{entries} && defined($map->{entries}->{$source});
 108
 109     return $map->{default} if $map->{default};
 110
 111     # identity (fallback)
 112     return $source;
 113 }
 114
 115 PVE::JSONSchema::register_standard_option('pve-targetstorage', {
 116     description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
 117     type => 'string',
 118     format => 'storagepair-list',
 119     optional => 1,
 120 });
 121
 122 #no warnings 'redefine';
 123
 124 sub cgroups_write {
 125    my ($controller, $vmid, $option, $value) = @_;
 126
 127    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 128    PVE::ProcFSTools::write_proc_entry($path, $value);
 129
 130 }
 131
 132 my $nodename_cache;
 133 sub nodename {
 134     $nodename_cache //= PVE::INotify::nodename();
 135     return $nodename_cache;
 136 }
 137
 138 my $watchdog_fmt = {
 139     model => {
 140         default_key => 1,
 141         type => 'string',
 142         enum => [qw(i6300esb ib700)],
 143         description => "Watchdog type to emulate.",
 144         default => 'i6300esb',
 145         optional => 1,
 146     },
 147     action => {
 148         type => 'string',
 149         enum => [qw(reset shutdown poweroff pause debug none)],
 150         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 151         optional => 1,
 152     },
 153 };
 154 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 155
 156 my $agent_fmt = {
 157     enabled => {
 158         description => "Enable/disable Qemu GuestAgent.",
 159         type => 'boolean',
 160         default => 0,
 161         default_key => 1,
 162     },
 163     fstrim_cloned_disks => {
 164         description => "Run fstrim after cloning/moving a disk.",
 165         type => 'boolean',
 166         optional => 1,
 167         default => 0
 168     },
 169     type => {
 170         description => "Select the agent type",
 171         type => 'string',
 172         default => 'virtio',
 173         optional => 1,
 174         enum => [qw(virtio isa)],
 175     },
 176 };
 177
 178 my $vga_fmt = {
 179     type => {
 180         description => "Select the VGA type.",
 181         type => 'string',
 182         default => 'std',
 183         optional => 1,
 184         default_key => 1,
 185         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 186     },
 187     memory => {
 188         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 189         type => 'integer',
 190         optional => 1,
 191         minimum => 4,
 192         maximum => 512,
 193     },
 194 };
 195
 196 my $ivshmem_fmt = {
 197     size => {
 198         type => 'integer',
 199         minimum => 1,
 200         description => "The size of the file in MB.",
 201     },
 202     name => {
 203         type => 'string',
 204         pattern => '[a-zA-Z0-9\-]+',
 205         optional => 1,
 206         format_description => 'string',
 207         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 208     },
 209 };
 210
 211 my $audio_fmt = {
 212     device => {
 213         type => 'string',
 214         enum => [qw(ich9-intel-hda intel-hda AC97)],
 215         description =>  "Configure an audio device."
 216     },
 217     driver =>  {
 218         type => 'string',
 219         enum => ['spice'],
 220         default => 'spice',
 221         optional => 1,
 222         description => "Driver backend for the audio device."
 223     },
 224 };
 225
 226 my $spice_enhancements_fmt = {
 227     foldersharing => {
 228         type => 'boolean',
 229         optional => 1,
 230         default => '0',
 231         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 232     },
 233     videostreaming =>  {
 234         type => 'string',
 235         enum => ['off', 'all', 'filter'],
 236         default => 'off',
 237         optional => 1,
 238         description => "Enable video streaming. Uses compression for detected video streams."
 239     },
 240 };
 241
 242 my $rng_fmt = {
 243     source => {
 244         type => 'string',
 245         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 246         default_key => 1,
 247         description => "The file on the host to gather entropy from. In most"
 248                      . " cases /dev/urandom should be preferred over /dev/random"
 249                      . " to avoid entropy-starvation issues on the host. Using"
 250                      . " urandom does *not* decrease security in any meaningful"
 251                      . " way, as it's still seeded from real entropy, and the"
 252                      . " bytes provided will most likely be mixed with real"
 253                      . " entropy on the guest as well. /dev/hwrng can be used"
 254                      . " to pass through a hardware RNG from the host.",
 255     },
 256     max_bytes => {
 257         type => 'integer',
 258         description => "Maximum bytes of entropy injected into the guest every"
 259                      . " 'period' milliseconds. Prefer a lower value when using"
 260                      . " /dev/random as source. Use 0 to disable limiting"
 261                      . " (potentially dangerous!).",
 262         optional => 1,
 263
 264         # default is 1 KiB/s, provides enough entropy to the guest to avoid
 265         # boot-starvation issues (e.g. systemd etc...) while allowing no chance
 266         # of overwhelming the host, provided we're reading from /dev/urandom
 267         default => 1024,
 268     },
 269     period => {
 270         type => 'integer',
 271         description => "Every 'period' milliseconds the entropy-injection quota"
 272                      . " is reset, allowing the guest to retrieve another"
 273                      . " 'max_bytes' of entropy.",
 274         optional => 1,
 275         default => 1000,
 276     },
 277 };
 278
 279 my $confdesc = {
 280     onboot => {
 281         optional => 1,
 282         type => 'boolean',
 283         description => "Specifies whether a VM will be started during system bootup.",
 284         default => 0,
 285     },
 286     autostart => {
 287         optional => 1,
 288         type => 'boolean',
 289         description => "Automatic restart after crash (currently ignored).",
 290         default => 0,
 291     },
 292     hotplug => {
 293         optional => 1,
 294         type => 'string', format => 'pve-hotplug-features',
 295         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 296         default => 'network,disk,usb',
 297     },
 298     reboot => {
 299         optional => 1,
 300         type => 'boolean',
 301         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 302         default => 1,
 303     },
 304     lock => {
 305         optional => 1,
 306         type => 'string',
 307         description => "Lock/unlock the VM.",
 308         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 309     },
 310     cpulimit => {
 311         optional => 1,
 312         type => 'number',
 313         description => "Limit of CPU usage.",
 314         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 315         minimum => 0,
 316         maximum => 128,
 317         default => 0,
 318     },
 319     cpuunits => {
 320         optional => 1,
 321         type => 'integer',
 322         description => "CPU weight for a VM.",
 323         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 324         minimum => 2,
 325         maximum => 262144,
 326         default => 1024,
 327     },
 328     memory => {
 329         optional => 1,
 330         type => 'integer',
 331         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 332         minimum => 16,
 333         default => 512,
 334     },
 335     balloon => {
 336         optional => 1,
 337         type => 'integer',
 338         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 339         minimum => 0,
 340     },
 341     shares => {
 342         optional => 1,
 343         type => 'integer',
 344         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 345         minimum => 0,
 346         maximum => 50000,
 347         default => 1000,
 348     },
 349     keyboard => {
 350         optional => 1,
 351         type => 'string',
 352         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 353                        "It should not be necessary to set it.",
 354         enum => PVE::Tools::kvmkeymaplist(),
 355         default => undef,
 356     },
 357     name => {
 358         optional => 1,
 359         type => 'string', format => 'dns-name',
 360         description => "Set a name for the VM. Only used on the configuration web interface.",
 361     },
 362     scsihw => {
 363         optional => 1,
 364         type => 'string',
 365         description => "SCSI controller model",
 366         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 367         default => 'lsi',
 368     },
 369     description => {
 370         optional => 1,
 371         type => 'string',
 372         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 373     },
 374     ostype => {
 375         optional => 1,
 376         type => 'string',
 377         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 378         description => "Specify guest operating system.",
 379         verbose_description => <<EODESC,
 380 Specify guest operating system. This is used to enable special
 381 optimization/features for specific operating systems:
 382
 383 [horizontal]
 384 other;; unspecified OS
 385 wxp;; Microsoft Windows XP
 386 w2k;; Microsoft Windows 2000
 387 w2k3;; Microsoft Windows 2003
 388 w2k8;; Microsoft Windows 2008
 389 wvista;; Microsoft Windows Vista
 390 win7;; Microsoft Windows 7
 391 win8;; Microsoft Windows 8/2012/2012r2
 392 win10;; Microsoft Windows 10/2016
 393 l24;; Linux 2.4 Kernel
 394 l26;; Linux 2.6 - 5.X Kernel
 395 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 396 EODESC
 397     },
 398     boot => {
 399         optional => 1,
 400         type => 'string',
 401         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 402         pattern => '[acdn]{1,4}',
 403         default => 'cdn',
 404     },
 405     bootdisk => {
 406         optional => 1,
 407         type => 'string', format => 'pve-qm-bootdisk',
 408         description => "Enable booting from specified disk.",
 409         pattern => '(ide|sata|scsi|virtio)\d+',
 410     },
 411     smp => {
 412         optional => 1,
 413         type => 'integer',
 414         description => "The number of CPUs. Please use option -sockets instead.",
 415         minimum => 1,
 416         default => 1,
 417     },
 418     sockets => {
 419         optional => 1,
 420         type => 'integer',
 421         description => "The number of CPU sockets.",
 422         minimum => 1,
 423         default => 1,
 424     },
 425     cores => {
 426         optional => 1,
 427         type => 'integer',
 428         description => "The number of cores per socket.",
 429         minimum => 1,
 430         default => 1,
 431     },
 432     numa => {
 433         optional => 1,
 434         type => 'boolean',
 435         description => "Enable/disable NUMA.",
 436         default => 0,
 437     },
 438     hugepages => {
 439         optional => 1,
 440         type => 'string',
 441         description => "Enable/disable hugepages memory.",
 442         enum => [qw(any 2 1024)],
 443     },
 444     vcpus => {
 445         optional => 1,
 446         type => 'integer',
 447         description => "Number of hotplugged vcpus.",
 448         minimum => 1,
 449         default => 0,
 450     },
 451     acpi => {
 452         optional => 1,
 453         type => 'boolean',
 454         description => "Enable/disable ACPI.",
 455         default => 1,
 456     },
 457     agent => {
 458         optional => 1,
 459         description => "Enable/disable Qemu GuestAgent and its properties.",
 460         type => 'string',
 461         format => $agent_fmt,
 462     },
 463     kvm => {
 464         optional => 1,
 465         type => 'boolean',
 466         description => "Enable/disable KVM hardware virtualization.",
 467         default => 1,
 468     },
 469     tdf => {
 470         optional => 1,
 471         type => 'boolean',
 472         description => "Enable/disable time drift fix.",
 473         default => 0,
 474     },
 475     localtime => {
 476         optional => 1,
 477         type => 'boolean',
 478         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 479     },
 480     freeze => {
 481         optional => 1,
 482         type => 'boolean',
 483         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 484     },
 485     vga => {
 486         optional => 1,
 487         type => 'string', format => $vga_fmt,
 488         description => "Configure the VGA hardware.",
 489         verbose_description => "Configure the VGA Hardware. If you want to use ".
 490             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 491             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 492             "is 'std' for all OS types besides some Windows versions (XP and " .
 493             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 494             "display server. For win* OS you can select how many independent " .
 495             "displays you want, Linux guests can add displays them self.\n".
 496             "You can also run without any graphic card, using a serial device as terminal.",
 497     },
 498     watchdog => {
 499         optional => 1,
 500         type => 'string', format => 'pve-qm-watchdog',
 501         description => "Create a virtual hardware watchdog device.",
 502         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 503             " (by a guest action), the watchdog must be periodically polled " .
 504             "by an agent inside the guest or else the watchdog will reset " .
 505             "the guest (or execute the respective action specified)",
 506     },
 507     startdate => {
 508         optional => 1,
 509         type => 'string',
 510         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 511         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 512         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 513         default => 'now',
 514     },
 515     startup =>  get_standard_option('pve-startup-order'),
 516     template => {
 517         optional => 1,
 518         type => 'boolean',
 519         description => "Enable/disable Template.",
 520         default => 0,
 521     },
 522     args => {
 523         optional => 1,
 524         type => 'string',
 525         description => "Arbitrary arguments passed to kvm.",
 526         verbose_description => <<EODESCR,
 527 Arbitrary arguments passed to kvm, for example:
 528
 529 args: -no-reboot -no-hpet
 530
 531 NOTE: this option is for experts only.
 532 EODESCR
 533     },
 534     tablet => {
 535         optional => 1,
 536         type => 'boolean',
 537         default => 1,
 538         description => "Enable/disable the USB tablet device.",
 539         verbose_description => "Enable/disable the USB tablet device. This device is " .
 540             "usually needed to allow absolute mouse positioning with VNC. " .
 541             "Else the mouse runs out of sync with normal VNC clients. " .
 542             "If you're running lots of console-only guests on one host, " .
 543             "you may consider disabling this to save some context switches. " .
 544             "This is turned off by default if you use spice (-vga=qxl).",
 545     },
 546     migrate_speed => {
 547         optional => 1,
 548         type => 'integer',
 549         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 550         minimum => 0,
 551         default => 0,
 552     },
 553     migrate_downtime => {
 554         optional => 1,
 555         type => 'number',
 556         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 557         minimum => 0,
 558         default => 0.1,
 559     },
 560     cdrom => {
 561         optional => 1,
 562         type => 'string', format => 'pve-qm-ide',
 563         typetext => '<volume>',
 564         description => "This is an alias for option -ide2",
 565     },
 566     cpu => {
 567         optional => 1,
 568         description => "Emulated CPU type.",
 569         type => 'string',
 570         format => 'pve-vm-cpu-conf',
 571     },
 572     parent => get_standard_option('pve-snapshot-name', {
 573         optional => 1,
 574         description => "Parent snapshot name. This is used internally, and should not be modified.",
 575     }),
 576     snaptime => {
 577         optional => 1,
 578         description => "Timestamp for snapshots.",
 579         type => 'integer',
 580         minimum => 0,
 581     },
 582     vmstate => {
 583         optional => 1,
 584         type => 'string', format => 'pve-volume-id',
 585         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 586     },
 587     vmstatestorage => get_standard_option('pve-storage-id', {
 588         description => "Default storage for VM state volumes/files.",
 589         optional => 1,
 590     }),
 591     runningmachine => get_standard_option('pve-qemu-machine', {
 592         description => "Specifies the QEMU machine type of the running vm. This is used internally for snapshots.",
 593     }),
 594     runningcpu => {
 595         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used internally for snapshots.",
 596         optional => 1,
 597         type => 'string',
 598         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 599         format_description => 'QEMU -cpu parameter'
 600     },
 601     machine => get_standard_option('pve-qemu-machine'),
 602     arch => {
 603         description => "Virtual processor architecture. Defaults to the host.",
 604         optional => 1,
 605         type => 'string',
 606         enum => [qw(x86_64 aarch64)],
 607     },
 608     smbios1 => {
 609         description => "Specify SMBIOS type 1 fields.",
 610         type => 'string', format => 'pve-qm-smbios1',
 611         maxLength => 512,
 612         optional => 1,
 613     },
 614     protection => {
 615         optional => 1,
 616         type => 'boolean',
 617         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 618         default => 0,
 619     },
 620     bios => {
 621         optional => 1,
 622         type => 'string',
 623         enum => [ qw(seabios ovmf) ],
 624         description => "Select BIOS implementation.",
 625         default => 'seabios',
 626     },
 627     vmgenid => {
 628         type => 'string',
 629         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 630         format_description => 'UUID',
 631         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 632         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 633             " 128-bit integer value identifier to the guest OS. This allows to".
 634             " notify the guest operating system when the virtual machine is".
 635             " executed with a different configuration (e.g. snapshot execution".
 636             " or creation from a template). The guest operating system notices".
 637             " the change, and is then able to react as appropriate by marking".
 638             " its copies of distributed databases as dirty, re-initializing its".
 639             " random number generator, etc.\n".
 640             "Note that auto-creation only works when done throug API/CLI create".
 641             " or update methods, but not when manually editing the config file.",
 642         default => "1 (autogenerated)",
 643         optional => 1,
 644     },
 645     hookscript => {
 646         type => 'string',
 647         format => 'pve-volume-id',
 648         optional => 1,
 649         description => "Script that will be executed during various steps in the vms lifetime.",
 650     },
 651     ivshmem => {
 652         type => 'string',
 653         format => $ivshmem_fmt,
 654         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 655         optional => 1,
 656     },
 657     audio0 => {
 658         type => 'string',
 659         format => $audio_fmt,
 660         description => "Configure a audio device, useful in combination with QXL/Spice.",
 661         optional => 1
 662     },
 663     spice_enhancements => {
 664         type => 'string',
 665         format => $spice_enhancements_fmt,
 666         description => "Configure additional enhancements for SPICE.",
 667         optional => 1
 668     },
 669     tags => {
 670         type => 'string', format => 'pve-tag-list',
 671         description => 'Tags of the VM. This is only meta information.',
 672         optional => 1,
 673     },
 674     rng0 => {
 675         type => 'string',
 676         format => $rng_fmt,
 677         description => "Configure a VirtIO-based Random Number Generator.",
 678         optional => 1,
 679     },
 680 };
 681
 682 my $cicustom_fmt = {
 683     meta => {
 684         type => 'string',
 685         optional => 1,
 686         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 687         format => 'pve-volume-id',
 688         format_description => 'volume',
 689     },
 690     network => {
 691         type => 'string',
 692         optional => 1,
 693         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 694         format => 'pve-volume-id',
 695         format_description => 'volume',
 696     },
 697     user => {
 698         type => 'string',
 699         optional => 1,
 700         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 701         format => 'pve-volume-id',
 702         format_description => 'volume',
 703     },
 704 };
 705 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 706
 707 my $confdesc_cloudinit = {
 708     citype => {
 709         optional => 1,
 710         type => 'string',
 711         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 712         enum => ['configdrive2', 'nocloud'],
 713     },
 714     ciuser => {
 715         optional => 1,
 716         type => 'string',
 717         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 718     },
 719     cipassword => {
 720         optional => 1,
 721         type => 'string',
 722         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 723     },
 724     cicustom => {
 725         optional => 1,
 726         type => 'string',
 727         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 728         format => 'pve-qm-cicustom',
 729     },
 730     searchdomain => {
 731         optional => 1,
 732         type => 'string',
 733         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 734     },
 735     nameserver => {
 736         optional => 1,
 737         type => 'string', format => 'address-list',
 738         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 739     },
 740     sshkeys => {
 741         optional => 1,
 742         type => 'string',
 743         format => 'urlencoded',
 744         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 745     },
 746 };
 747
 748 # what about other qemu settings ?
 749 #cpu => 'string',
 750 #machine => 'string',
 751 #fda => 'file',
 752 #fdb => 'file',
 753 #mtdblock => 'file',
 754 #sd => 'file',
 755 #pflash => 'file',
 756 #snapshot => 'bool',
 757 #bootp => 'file',
 758 ##tftp => 'dir',
 759 ##smb => 'dir',
 760 #kernel => 'file',
 761 #append => 'string',
 762 #initrd => 'file',
 763 ##soundhw => 'string',
 764
 765 while (my ($k, $v) = each %$confdesc) {
 766     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 767 }
 768
 769 my $MAX_USB_DEVICES = 5;
 770 my $MAX_NETS = 32;
 771 my $MAX_SERIAL_PORTS = 4;
 772 my $MAX_PARALLEL_PORTS = 3;
 773 my $MAX_NUMA = 8;
 774
 775 my $numa_fmt = {
 776     cpus => {
 777         type => "string",
 778         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 779         description => "CPUs accessing this NUMA node.",
 780         format_description => "id[-id];...",
 781     },
 782     memory => {
 783         type => "number",
 784         description => "Amount of memory this NUMA node provides.",
 785         optional => 1,
 786     },
 787     hostnodes => {
 788         type => "string",
 789         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 790         description => "Host NUMA nodes to use.",
 791         format_description => "id[-id];...",
 792         optional => 1,
 793     },
 794     policy => {
 795         type => 'string',
 796         enum => [qw(preferred bind interleave)],
 797         description => "NUMA allocation policy.",
 798         optional => 1,
 799     },
 800 };
 801 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 802 my $numadesc = {
 803     optional => 1,
 804     type => 'string', format => $numa_fmt,
 805     description => "NUMA topology.",
 806 };
 807 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 808
 809 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 810     $confdesc->{"numa$i"} = $numadesc;
 811 }
 812
 813 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 814                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 815                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 816 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 817
 818 my $net_fmt_bridge_descr = <<__EOD__;
 819 Bridge to attach the network device to. The Proxmox VE standard bridge
 820 is called 'vmbr0'.
 821
 822 If you do not specify a bridge, we create a kvm user (NATed) network
 823 device, which provides DHCP and DNS services. The following addresses
 824 are used:
 825
 826  10.0.2.2   Gateway
 827  10.0.2.3   DNS Server
 828  10.0.2.4   SMB Server
 829
 830 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 831 __EOD__
 832
 833 my $net_fmt = {
 834     macaddr  => get_standard_option('mac-addr', {
 835         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 836     }),
 837     model => {
 838         type => 'string',
 839         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 840         enum => $nic_model_list,
 841         default_key => 1,
 842     },
 843     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 844     bridge => {
 845         type => 'string',
 846         description => $net_fmt_bridge_descr,
 847         format_description => 'bridge',
 848         pattern => '[-_.\w\d]+',
 849         optional => 1,
 850     },
 851     queues => {
 852         type => 'integer',
 853         minimum => 0, maximum => 16,
 854         description => 'Number of packet queues to be used on the device.',
 855         optional => 1,
 856     },
 857     rate => {
 858         type => 'number',
 859         minimum => 0,
 860         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 861         optional => 1,
 862     },
 863     tag => {
 864         type => 'integer',
 865         minimum => 1, maximum => 4094,
 866         description => 'VLAN tag to apply to packets on this interface.',
 867         optional => 1,
 868     },
 869     trunks => {
 870         type => 'string',
 871         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 872         description => 'VLAN trunks to pass through this interface.',
 873         format_description => 'vlanid[;vlanid...]',
 874         optional => 1,
 875     },
 876     firewall => {
 877         type => 'boolean',
 878         description => 'Whether this interface should be protected by the firewall.',
 879         optional => 1,
 880     },
 881     link_down => {
 882         type => 'boolean',
 883         description => 'Whether this interface should be disconnected (like pulling the plug).',
 884         optional => 1,
 885     },
 886     mtu => {
 887         type => 'integer',
 888         minimum => 1, maximum => 65520,
 889         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 890         optional => 1,
 891     },
 892 };
 893
 894 my $netdesc = {
 895     optional => 1,
 896     type => 'string', format => $net_fmt,
 897     description => "Specify network devices.",
 898 };
 899
 900 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 901
 902 my $ipconfig_fmt = {
 903     ip => {
 904         type => 'string',
 905         format => 'pve-ipv4-config',
 906         format_description => 'IPv4Format/CIDR',
 907         description => 'IPv4 address in CIDR format.',
 908         optional => 1,
 909         default => 'dhcp',
 910     },
 911     gw => {
 912         type => 'string',
 913         format => 'ipv4',
 914         format_description => 'GatewayIPv4',
 915         description => 'Default gateway for IPv4 traffic.',
 916         optional => 1,
 917         requires => 'ip',
 918     },
 919     ip6 => {
 920         type => 'string',
 921         format => 'pve-ipv6-config',
 922         format_description => 'IPv6Format/CIDR',
 923         description => 'IPv6 address in CIDR format.',
 924         optional => 1,
 925         default => 'dhcp',
 926     },
 927     gw6 => {
 928         type => 'string',
 929         format => 'ipv6',
 930         format_description => 'GatewayIPv6',
 931         description => 'Default gateway for IPv6 traffic.',
 932         optional => 1,
 933         requires => 'ip6',
 934     },
 935 };
 936 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 937 my $ipconfigdesc = {
 938     optional => 1,
 939     type => 'string', format => 'pve-qm-ipconfig',
 940     description => <<'EODESCR',
 941 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 942
 943 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 944
 945 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 946 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 947
 948 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 949 EODESCR
 950 };
 951 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 952
 953 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 954     $confdesc->{"net$i"} = $netdesc;
 955     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 956 }
 957
 958 foreach my $key (keys %$confdesc_cloudinit) {
 959     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 960 }
 961
 962 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 963 sub verify_volume_id_or_qm_path {
 964     my ($volid, $noerr) = @_;
 965
 966     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 967         return $volid;
 968     }
 969
 970     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 971     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 972     if ($@) {
 973         return undef if $noerr;
 974         die $@;
 975     }
 976     return $volid;
 977 }
 978
 979 my $usb_fmt = {
 980     host => {
 981         default_key => 1,
 982         type => 'string', format => 'pve-qm-usb-device',
 983         format_description => 'HOSTUSBDEVICE|spice',
 984         description => <<EODESCR,
 985 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
 986
 987  'bus-port(.port)*' (decimal numbers) or
 988  'vendor_id:product_id' (hexadeciaml numbers) or
 989  'spice'
 990
 991 You can use the 'lsusb -t' command to list existing usb devices.
 992
 993 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 994
 995 The value 'spice' can be used to add a usb redirection devices for spice.
 996 EODESCR
 997     },
 998     usb3 => {
 999         optional => 1,
1000         type => 'boolean',
1001         description => "Specifies whether if given host option is a USB3 device or port.",
1002         default => 0,
1003     },
1004 };
1005
1006 my $usbdesc = {
1007     optional => 1,
1008     type => 'string', format => $usb_fmt,
1009     description => "Configure an USB device (n is 0 to 4).",
1010 };
1011 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1012
1013 my $serialdesc = {
1014         optional => 1,
1015         type => 'string',
1016         pattern => '(/dev/.+|socket)',
1017         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1018         verbose_description =>  <<EODESCR,
1019 Create a serial device inside the VM (n is 0 to 3), and pass through a
1020 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1021 host side (use 'qm terminal' to open a terminal connection).
1022
1023 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1024
1025 CAUTION: Experimental! User reported problems with this option.
1026 EODESCR
1027 };
1028
1029 my $paralleldesc= {
1030         optional => 1,
1031         type => 'string',
1032         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1033         description =>  "Map host parallel devices (n is 0 to 2).",
1034         verbose_description =>  <<EODESCR,
1035 Map host parallel devices (n is 0 to 2).
1036
1037 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1038
1039 CAUTION: Experimental! User reported problems with this option.
1040 EODESCR
1041 };
1042
1043 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1044     $confdesc->{"parallel$i"} = $paralleldesc;
1045 }
1046
1047 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1048     $confdesc->{"serial$i"} = $serialdesc;
1049 }
1050
1051 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1052     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1053 }
1054
1055 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1056     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1057 }
1058
1059 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1060     $confdesc->{"usb$i"} = $usbdesc;
1061 }
1062
1063 my $kvm_api_version = 0;
1064
1065 sub kvm_version {
1066     return $kvm_api_version if $kvm_api_version;
1067
1068     open my $fh, '<', '/dev/kvm'
1069         or return undef;
1070
1071     # 0xae00 => KVM_GET_API_VERSION
1072     $kvm_api_version = ioctl($fh, 0xae00, 0);
1073
1074     return $kvm_api_version;
1075 }
1076
1077 my $kvm_user_version = {};
1078 my $kvm_mtime = {};
1079
1080 sub kvm_user_version {
1081     my ($binary) = @_;
1082
1083     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1084     my $st = stat($binary);
1085
1086     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1087     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1088         $cachedmtime == $st->mtime;
1089
1090     $kvm_user_version->{$binary} = 'unknown';
1091     $kvm_mtime->{$binary} = $st->mtime;
1092
1093     my $code = sub {
1094         my $line = shift;
1095         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1096             $kvm_user_version->{$binary} = $2;
1097         }
1098     };
1099
1100     eval { run_command([$binary, '--version'], outfunc => $code); };
1101     warn $@ if $@;
1102
1103     return $kvm_user_version->{$binary};
1104
1105 }
1106
1107 sub kernel_has_vhost_net {
1108     return -c '/dev/vhost-net';
1109 }
1110
1111 sub option_exists {
1112     my $key = shift;
1113     return defined($confdesc->{$key});
1114 }
1115
1116 my $cdrom_path;
1117 sub get_cdrom_path {
1118
1119     return  $cdrom_path if $cdrom_path;
1120
1121     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1122     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1123     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1124 }
1125
1126 sub get_iso_path {
1127     my ($storecfg, $vmid, $cdrom) = @_;
1128
1129     if ($cdrom eq 'cdrom') {
1130         return get_cdrom_path();
1131     } elsif ($cdrom eq 'none') {
1132         return '';
1133     } elsif ($cdrom =~ m|^/|) {
1134         return $cdrom;
1135     } else {
1136         return PVE::Storage::path($storecfg, $cdrom);
1137     }
1138 }
1139
1140 # try to convert old style file names to volume IDs
1141 sub filename_to_volume_id {
1142     my ($vmid, $file, $media) = @_;
1143
1144      if (!($file eq 'none' || $file eq 'cdrom' ||
1145           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1146
1147         return undef if $file =~ m|/|;
1148
1149         if ($media && $media eq 'cdrom') {
1150             $file = "local:iso/$file";
1151         } else {
1152             $file = "local:$vmid/$file";
1153         }
1154     }
1155
1156     return $file;
1157 }
1158
1159 sub verify_media_type {
1160     my ($opt, $vtype, $media) = @_;
1161
1162     return if !$media;
1163
1164     my $etype;
1165     if ($media eq 'disk') {
1166         $etype = 'images';
1167     } elsif ($media eq 'cdrom') {
1168         $etype = 'iso';
1169     } else {
1170         die "internal error";
1171     }
1172
1173     return if ($vtype eq $etype);
1174
1175     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1176 }
1177
1178 sub cleanup_drive_path {
1179     my ($opt, $storecfg, $drive) = @_;
1180
1181     # try to convert filesystem paths to volume IDs
1182
1183     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1184         ($drive->{file} !~ m|^/dev/.+|) &&
1185         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1186         ($drive->{file} !~ m/^\d+$/)) {
1187         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1188         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1189         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1190         verify_media_type($opt, $vtype, $drive->{media});
1191         $drive->{file} = $volid;
1192     }
1193
1194     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1195 }
1196
1197 sub parse_hotplug_features {
1198     my ($data) = @_;
1199
1200     my $res = {};
1201
1202     return $res if $data eq '0';
1203
1204     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1205
1206     foreach my $feature (PVE::Tools::split_list($data)) {
1207         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1208             $res->{$1} = 1;
1209         } else {
1210             die "invalid hotplug feature '$feature'\n";
1211         }
1212     }
1213     return $res;
1214 }
1215
1216 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1217 sub pve_verify_hotplug_features {
1218     my ($value, $noerr) = @_;
1219
1220     return $value if parse_hotplug_features($value);
1221
1222     return undef if $noerr;
1223
1224     die "unable to parse hotplug option\n";
1225 }
1226
1227 sub scsi_inquiry {
1228     my($fh, $noerr) = @_;
1229
1230     my $SG_IO = 0x2285;
1231     my $SG_GET_VERSION_NUM = 0x2282;
1232
1233     my $versionbuf = "\x00" x 8;
1234     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1235     if (!$ret) {
1236         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1237         return undef;
1238     }
1239     my $version = unpack("I", $versionbuf);
1240     if ($version < 30000) {
1241         die "scsi generic interface too old\n"  if !$noerr;
1242         return undef;
1243     }
1244
1245     my $buf = "\x00" x 36;
1246     my $sensebuf = "\x00" x 8;
1247     my $cmd = pack("C x3 C x1", 0x12, 36);
1248
1249     # see /usr/include/scsi/sg.h
1250     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1251
1252     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1253                       length($sensebuf), 0, length($buf), $buf,
1254                       $cmd, $sensebuf, 6000);
1255
1256     $ret = ioctl($fh, $SG_IO, $packet);
1257     if (!$ret) {
1258         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1259         return undef;
1260     }
1261
1262     my @res = unpack($sg_io_hdr_t, $packet);
1263     if ($res[17] || $res[18]) {
1264         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1265         return undef;
1266     }
1267
1268     my $res = {};
1269     (my $byte0, my $byte1, $res->{vendor},
1270      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1271
1272     $res->{removable} = $byte1 & 128 ? 1 : 0;
1273     $res->{type} = $byte0 & 31;
1274
1275     return $res;
1276 }
1277
1278 sub path_is_scsi {
1279     my ($path) = @_;
1280
1281     my $fh = IO::File->new("+<$path") || return undef;
1282     my $res = scsi_inquiry($fh, 1);
1283     close($fh);
1284
1285     return $res;
1286 }
1287
1288 sub print_tabletdevice_full {
1289     my ($conf, $arch) = @_;
1290
1291     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1292
1293     # we use uhci for old VMs because tablet driver was buggy in older qemu
1294     my $usbbus;
1295     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1296         $usbbus = 'ehci';
1297     } else {
1298         $usbbus = 'uhci';
1299     }
1300
1301     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1302 }
1303
1304 sub print_keyboarddevice_full {
1305     my ($conf, $arch, $machine) = @_;
1306
1307     return undef if $arch ne 'aarch64';
1308
1309     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1310 }
1311
1312 sub print_drivedevice_full {
1313     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1314
1315     my $device = '';
1316     my $maxdev = 0;
1317
1318     if ($drive->{interface} eq 'virtio') {
1319         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1320         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1321         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1322     } elsif ($drive->{interface} eq 'scsi') {
1323
1324         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1325         my $unit = $drive->{index} % $maxdev;
1326         my $devicetype = 'hd';
1327         my $path = '';
1328         if (drive_is_cdrom($drive)) {
1329             $devicetype = 'cd';
1330         } else {
1331             if ($drive->{file} =~ m|^/|) {
1332                 $path = $drive->{file};
1333                 if (my $info = path_is_scsi($path)) {
1334                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1335                         $devicetype = 'block';
1336                     } elsif ($info->{type} == 1) { # tape
1337                         $devicetype = 'generic';
1338                     }
1339                 }
1340             } else {
1341                  $path = PVE::Storage::path($storecfg, $drive->{file});
1342             }
1343
1344             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1345             my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
1346             if ($path =~ m/^iscsi\:\/\// &&
1347                !min_version($version, 4, 1)) {
1348                 $devicetype = 'generic';
1349             }
1350         }
1351
1352         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1353             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1354         } else {
1355             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1356         }
1357
1358         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1359             $device .= ",rotation_rate=1";
1360         }
1361         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1362
1363     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1364         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1365         my $controller = int($drive->{index} / $maxdev);
1366         my $unit = $drive->{index} % $maxdev;
1367         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1368
1369         $device = "ide-$devicetype";
1370         if ($drive->{interface} eq 'ide') {
1371             $device .= ",bus=ide.$controller,unit=$unit";
1372         } else {
1373             $device .= ",bus=ahci$controller.$unit";
1374         }
1375         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1376
1377         if ($devicetype eq 'hd') {
1378             if (my $model = $drive->{model}) {
1379                 $model = URI::Escape::uri_unescape($model);
1380                 $device .= ",model=$model";
1381             }
1382             if ($drive->{ssd}) {
1383                 $device .= ",rotation_rate=1";
1384             }
1385         }
1386         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1387     } elsif ($drive->{interface} eq 'usb') {
1388         die "implement me";
1389         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1390     } else {
1391         die "unsupported interface type";
1392     }
1393
1394     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1395
1396     if (my $serial = $drive->{serial}) {
1397         $serial = URI::Escape::uri_unescape($serial);
1398         $device .= ",serial=$serial";
1399     }
1400
1401
1402     return $device;
1403 }
1404
1405 sub get_initiator_name {
1406     my $initiator;
1407
1408     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1409     while (defined(my $line = <$fh>)) {
1410         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1411         $initiator = $1;
1412         last;
1413     }
1414     $fh->close();
1415
1416     return $initiator;
1417 }
1418
1419 sub print_drive_commandline_full {
1420     my ($storecfg, $vmid, $drive) = @_;
1421
1422     my $path;
1423     my $volid = $drive->{file};
1424     my $format;
1425
1426     if (drive_is_cdrom($drive)) {
1427         $path = get_iso_path($storecfg, $vmid, $volid);
1428     } else {
1429         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1430         if ($storeid) {
1431             $path = PVE::Storage::path($storecfg, $volid);
1432             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1433             $format = qemu_img_format($scfg, $volname);
1434         } else {
1435             $path = $volid;
1436             $format = "raw";
1437         }
1438    }
1439
1440     my $opts = '';
1441     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1442     foreach my $o (@qemu_drive_options) {
1443         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1444     }
1445
1446     # snapshot only accepts on|off
1447     if (defined($drive->{snapshot})) {
1448         my $v = $drive->{snapshot} ? 'on' : 'off';
1449         $opts .= ",snapshot=$v";
1450     }
1451
1452     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1453         my ($dir, $qmpname) = @$type;
1454         if (my $v = $drive->{"mbps$dir"}) {
1455             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1456         }
1457         if (my $v = $drive->{"mbps${dir}_max"}) {
1458             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1459         }
1460         if (my $v = $drive->{"bps${dir}_max_length"}) {
1461             $opts .= ",throttling.bps$qmpname-max-length=$v";
1462         }
1463         if (my $v = $drive->{"iops${dir}"}) {
1464             $opts .= ",throttling.iops$qmpname=$v";
1465         }
1466         if (my $v = $drive->{"iops${dir}_max"}) {
1467             $opts .= ",throttling.iops$qmpname-max=$v";
1468         }
1469         if (my $v = $drive->{"iops${dir}_max_length"}) {
1470             $opts .= ",throttling.iops$qmpname-max-length=$v";
1471         }
1472     }
1473
1474     $opts .= ",format=$format" if $format && !$drive->{format};
1475
1476     my $cache_direct = 0;
1477
1478     if (my $cache = $drive->{cache}) {
1479         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1480     } elsif (!drive_is_cdrom($drive)) {
1481         $opts .= ",cache=none";
1482         $cache_direct = 1;
1483     }
1484
1485     # aio native works only with O_DIRECT
1486     if (!$drive->{aio}) {
1487         if($cache_direct) {
1488             $opts .= ",aio=native";
1489         } else {
1490             $opts .= ",aio=threads";
1491         }
1492     }
1493
1494     if (!drive_is_cdrom($drive)) {
1495         my $detectzeroes;
1496         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1497             $detectzeroes = 'off';
1498         } elsif ($drive->{discard}) {
1499             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1500         } else {
1501             # This used to be our default with discard not being specified:
1502             $detectzeroes = 'on';
1503         }
1504         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1505     }
1506
1507     my $pathinfo = $path ? "file=$path," : '';
1508
1509     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1510 }
1511
1512 sub print_netdevice_full {
1513     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1514
1515     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1516
1517     my $device = $net->{model};
1518     if ($net->{model} eq 'virtio') {
1519          $device = 'virtio-net-pci';
1520      };
1521
1522     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1523     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1524     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1525         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1526         my $vectors = $net->{queues} * 2 + 2;
1527         $tmpstr .= ",vectors=$vectors,mq=on";
1528     }
1529     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1530
1531     if (my $mtu = $net->{mtu}) {
1532         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1533             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1534             if ($mtu == 1) {
1535                  $mtu = $bridge_mtu;
1536             } elsif ($mtu < 576) {
1537                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1538             } elsif ($mtu > $bridge_mtu) {
1539                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1540             }
1541             $tmpstr .= ",host_mtu=$mtu";
1542         } else {
1543             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1544         }
1545     }
1546
1547     if ($use_old_bios_files) {
1548         my $romfile;
1549         if ($device eq 'virtio-net-pci') {
1550             $romfile = 'pxe-virtio.rom';
1551         } elsif ($device eq 'e1000') {
1552             $romfile = 'pxe-e1000.rom';
1553         } elsif ($device eq 'ne2k') {
1554             $romfile = 'pxe-ne2k_pci.rom';
1555         } elsif ($device eq 'pcnet') {
1556             $romfile = 'pxe-pcnet.rom';
1557         } elsif ($device eq 'rtl8139') {
1558             $romfile = 'pxe-rtl8139.rom';
1559         }
1560         $tmpstr .= ",romfile=$romfile" if $romfile;
1561     }
1562
1563     return $tmpstr;
1564 }
1565
1566 sub print_netdev_full {
1567     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1568
1569     my $i = '';
1570     if ($netid =~ m/^net(\d+)$/) {
1571         $i = int($1);
1572     }
1573
1574     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1575
1576     my $ifname = "tap${vmid}i$i";
1577
1578     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1579     die "interface name '$ifname' is too long (max 15 character)\n"
1580         if length($ifname) >= 16;
1581
1582     my $vhostparam = '';
1583     if (is_native($arch)) {
1584         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1585     }
1586
1587     my $vmname = $conf->{name} || "vm$vmid";
1588
1589     my $netdev = "";
1590     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1591
1592     if ($net->{bridge}) {
1593         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1594     } else {
1595         $netdev = "type=user,id=$netid,hostname=$vmname";
1596     }
1597
1598     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1599
1600     return $netdev;
1601 }
1602
1603 my $vga_map = {
1604     'cirrus' => 'cirrus-vga',
1605     'std' => 'VGA',
1606     'vmware' => 'vmware-svga',
1607     'virtio' => 'virtio-vga',
1608 };
1609
1610 sub print_vga_device {
1611     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1612
1613     my $type = $vga_map->{$vga->{type}};
1614     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1615         $type = 'virtio-gpu';
1616     }
1617     my $vgamem_mb = $vga->{memory};
1618
1619     my $max_outputs = '';
1620     if ($qxlnum) {
1621         $type = $id ? 'qxl' : 'qxl-vga';
1622
1623         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1624             # set max outputs so linux can have up to 4 qxl displays with one device
1625             if (min_version($machine_version, 4, 1)) {
1626                 $max_outputs = ",max_outputs=4";
1627             }
1628         }
1629     }
1630
1631     die "no devicetype for $vga->{type}\n" if !$type;
1632
1633     my $memory = "";
1634     if ($vgamem_mb) {
1635         if ($vga->{type} eq 'virtio') {
1636             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1637             $memory = ",max_hostmem=$bytes";
1638         } elsif ($qxlnum) {
1639             # from https://www.spice-space.org/multiple-monitors.html
1640             $memory = ",vgamem_mb=$vga->{memory}";
1641             my $ram = $vgamem_mb * 4;
1642             my $vram = $vgamem_mb * 2;
1643             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1644         } else {
1645             $memory = ",vgamem_mb=$vga->{memory}";
1646         }
1647     } elsif ($qxlnum && $id) {
1648         $memory = ",ram_size=67108864,vram_size=33554432";
1649     }
1650
1651     my $edidoff = "";
1652     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1653         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1654     }
1655
1656     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1657     my $vgaid = "vga" . ($id // '');
1658     my $pciaddr;
1659
1660     if ($q35 && $vgaid eq 'vga') {
1661         # the first display uses pcie.0 bus on q35 machines
1662         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1663     } else {
1664         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1665     }
1666
1667     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1668 }
1669
1670 sub parse_number_sets {
1671     my ($set) = @_;
1672     my $res = [];
1673     foreach my $part (split(/;/, $set)) {
1674         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1675             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1676             push @$res, [ $1, $2 ];
1677         } else {
1678             die "invalid range: $part\n";
1679         }
1680     }
1681     return $res;
1682 }
1683
1684 sub parse_numa {
1685     my ($data) = @_;
1686
1687     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1688     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1689     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1690     return $res;
1691 }
1692
1693 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1694 sub parse_net {
1695     my ($data) = @_;
1696
1697     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1698     if ($@) {
1699         warn $@;
1700         return undef;
1701     }
1702     if (!defined($res->{macaddr})) {
1703         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1704         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1705     }
1706     return $res;
1707 }
1708
1709 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1710 sub parse_ipconfig {
1711     my ($data) = @_;
1712
1713     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
1714     if ($@) {
1715         warn $@;
1716         return undef;
1717     }
1718
1719     if ($res->{gw} && !$res->{ip}) {
1720         warn 'gateway specified without specifying an IP address';
1721         return undef;
1722     }
1723     if ($res->{gw6} && !$res->{ip6}) {
1724         warn 'IPv6 gateway specified without specifying an IPv6 address';
1725         return undef;
1726     }
1727     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1728         warn 'gateway specified together with DHCP';
1729         return undef;
1730     }
1731     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1732         # gw6 + auto/dhcp
1733         warn "IPv6 gateway specified together with $res->{ip6} address";
1734         return undef;
1735     }
1736
1737     if (!$res->{ip} && !$res->{ip6}) {
1738         return { ip => 'dhcp', ip6 => 'dhcp' };
1739     }
1740
1741     return $res;
1742 }
1743
1744 sub print_net {
1745     my $net = shift;
1746
1747     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1748 }
1749
1750 sub add_random_macs {
1751     my ($settings) = @_;
1752
1753     foreach my $opt (keys %$settings) {
1754         next if $opt !~ m/^net(\d+)$/;
1755         my $net = parse_net($settings->{$opt});
1756         next if !$net;
1757         $settings->{$opt} = print_net($net);
1758     }
1759 }
1760
1761 sub vm_is_volid_owner {
1762     my ($storecfg, $vmid, $volid) = @_;
1763
1764     if ($volid !~  m|^/|) {
1765         my ($path, $owner);
1766         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1767         if ($owner && ($owner == $vmid)) {
1768             return 1;
1769         }
1770     }
1771
1772     return undef;
1773 }
1774
1775 sub vmconfig_register_unused_drive {
1776     my ($storecfg, $vmid, $conf, $drive) = @_;
1777
1778     if (drive_is_cloudinit($drive)) {
1779         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1780         warn $@ if $@;
1781     } elsif (!drive_is_cdrom($drive)) {
1782         my $volid = $drive->{file};
1783         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1784             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1785         }
1786     }
1787 }
1788
1789 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1790 my $smbios1_fmt = {
1791     uuid => {
1792         type => 'string',
1793         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1794         format_description => 'UUID',
1795         description => "Set SMBIOS1 UUID.",
1796         optional => 1,
1797     },
1798     version => {
1799         type => 'string',
1800         pattern => '[A-Za-z0-9+\/]+={0,2}',
1801         format_description => 'Base64 encoded string',
1802         description => "Set SMBIOS1 version.",
1803         optional => 1,
1804     },
1805     serial => {
1806         type => 'string',
1807         pattern => '[A-Za-z0-9+\/]+={0,2}',
1808         format_description => 'Base64 encoded string',
1809         description => "Set SMBIOS1 serial number.",
1810         optional => 1,
1811     },
1812     manufacturer => {
1813         type => 'string',
1814         pattern => '[A-Za-z0-9+\/]+={0,2}',
1815         format_description => 'Base64 encoded string',
1816         description => "Set SMBIOS1 manufacturer.",
1817         optional => 1,
1818     },
1819     product => {
1820         type => 'string',
1821         pattern => '[A-Za-z0-9+\/]+={0,2}',
1822         format_description => 'Base64 encoded string',
1823         description => "Set SMBIOS1 product ID.",
1824         optional => 1,
1825     },
1826     sku => {
1827         type => 'string',
1828         pattern => '[A-Za-z0-9+\/]+={0,2}',
1829         format_description => 'Base64 encoded string',
1830         description => "Set SMBIOS1 SKU string.",
1831         optional => 1,
1832     },
1833     family => {
1834         type => 'string',
1835         pattern => '[A-Za-z0-9+\/]+={0,2}',
1836         format_description => 'Base64 encoded string',
1837         description => "Set SMBIOS1 family string.",
1838         optional => 1,
1839     },
1840     base64 => {
1841         type => 'boolean',
1842         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
1843         optional => 1,
1844     },
1845 };
1846
1847 sub parse_smbios1 {
1848     my ($data) = @_;
1849
1850     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
1851     warn $@ if $@;
1852     return $res;
1853 }
1854
1855 sub print_smbios1 {
1856     my ($smbios1) = @_;
1857     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1858 }
1859
1860 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1861
1862 sub parse_watchdog {
1863     my ($value) = @_;
1864
1865     return undef if !$value;
1866
1867     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
1868     warn $@ if $@;
1869     return $res;
1870 }
1871
1872 sub parse_guest_agent {
1873     my ($value) = @_;
1874
1875     return {} if !defined($value->{agent});
1876
1877     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
1878     warn $@ if $@;
1879
1880     # if the agent is disabled ignore the other potentially set properties
1881     return {} if !$res->{enabled};
1882     return $res;
1883 }
1884
1885 sub parse_vga {
1886     my ($value) = @_;
1887
1888     return {} if !$value;
1889     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
1890     warn $@ if $@;
1891     return $res;
1892 }
1893
1894 sub parse_rng {
1895     my ($value) = @_;
1896
1897     return undef if !$value;
1898
1899     my $res = eval { PVE::JSONSchema::parse_property_string($rng_fmt, $value) };
1900     warn $@ if $@;
1901     return $res;
1902 }
1903
1904 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1905 sub verify_usb_device {
1906     my ($value, $noerr) = @_;
1907
1908     return $value if parse_usb_device($value);
1909
1910     return undef if $noerr;
1911
1912     die "unable to parse usb device\n";
1913 }
1914
1915 # add JSON properties for create and set function
1916 sub json_config_properties {
1917     my $prop = shift;
1918
1919     foreach my $opt (keys %$confdesc) {
1920         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' ||
1921             $opt eq 'runningmachine' || $opt eq 'runningcpu';
1922         $prop->{$opt} = $confdesc->{$opt};
1923     }
1924
1925     return $prop;
1926 }
1927
1928 # return copy of $confdesc_cloudinit to generate documentation
1929 sub cloudinit_config_properties {
1930
1931     return dclone($confdesc_cloudinit);
1932 }
1933
1934 sub check_type {
1935     my ($key, $value) = @_;
1936
1937     die "unknown setting '$key'\n" if !$confdesc->{$key};
1938
1939     my $type = $confdesc->{$key}->{type};
1940
1941     if (!defined($value)) {
1942         die "got undefined value\n";
1943     }
1944
1945     if ($value =~ m/[\n\r]/) {
1946         die "property contains a line feed\n";
1947     }
1948
1949     if ($type eq 'boolean') {
1950         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1951         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1952         die "type check ('boolean') failed - got '$value'\n";
1953     } elsif ($type eq 'integer') {
1954         return int($1) if $value =~ m/^(\d+)$/;
1955         die "type check ('integer') failed - got '$value'\n";
1956     } elsif ($type eq 'number') {
1957         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1958         die "type check ('number') failed - got '$value'\n";
1959     } elsif ($type eq 'string') {
1960         if (my $fmt = $confdesc->{$key}->{format}) {
1961             PVE::JSONSchema::check_format($fmt, $value);
1962             return $value;
1963         }
1964         $value =~ s/^\"(.*)\"$/$1/;
1965         return $value;
1966     } else {
1967         die "internal error"
1968     }
1969 }
1970
1971 sub destroy_vm {
1972     my ($storecfg, $vmid, $skiplock, $replacement_conf) = @_;
1973
1974     my $conf = PVE::QemuConfig->load_config($vmid);
1975
1976     PVE::QemuConfig->check_lock($conf) if !$skiplock;
1977
1978     if ($conf->{template}) {
1979         # check if any base image is still used by a linked clone
1980         PVE::QemuConfig->foreach_volume($conf, sub {
1981                 my ($ds, $drive) = @_;
1982                 return if drive_is_cdrom($drive);
1983
1984                 my $volid = $drive->{file};
1985                 return if !$volid || $volid =~ m|^/|;
1986
1987                 die "base volume '$volid' is still in use by linked cloned\n"
1988                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
1989
1990         });
1991     }
1992
1993     # only remove disks owned by this VM
1994     PVE::QemuConfig->foreach_volume($conf, sub {
1995         my ($ds, $drive) = @_;
1996         return if drive_is_cdrom($drive, 1);
1997
1998         my $volid = $drive->{file};
1999         return if !$volid || $volid =~ m|^/|;
2000
2001         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2002         return if !$path || !$owner || ($owner != $vmid);
2003
2004         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2005         warn "Could not remove disk '$volid', check manually: $@" if $@;
2006     });
2007
2008     # also remove unused disk
2009     my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2010     PVE::Storage::foreach_volid($vmdisks, sub {
2011         my ($volid, $sid, $volname, $d) = @_;
2012         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2013         warn $@ if $@;
2014     });
2015
2016     if (defined $replacement_conf) {
2017         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2018     } else {
2019         PVE::QemuConfig->destroy_config($vmid);
2020     }
2021 }
2022
2023 sub parse_vm_config {
2024     my ($filename, $raw) = @_;
2025
2026     return undef if !defined($raw);
2027
2028     my $res = {
2029         digest => Digest::SHA::sha1_hex($raw),
2030         snapshots => {},
2031         pending => {},
2032     };
2033
2034     $filename =~ m|/qemu-server/(\d+)\.conf$|
2035         || die "got strange filename '$filename'";
2036
2037     my $vmid = $1;
2038
2039     my $conf = $res;
2040     my $descr;
2041     my $section = '';
2042
2043     my @lines = split(/\n/, $raw);
2044     foreach my $line (@lines) {
2045         next if $line =~ m/^\s*$/;
2046
2047         if ($line =~ m/^\[PENDING\]\s*$/i) {
2048             $section = 'pending';
2049             if (defined($descr)) {
2050                 $descr =~ s/\s+$//;
2051                 $conf->{description} = $descr;
2052             }
2053             $descr = undef;
2054             $conf = $res->{$section} = {};
2055             next;
2056
2057         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2058             $section = $1;
2059             if (defined($descr)) {
2060                 $descr =~ s/\s+$//;
2061                 $conf->{description} = $descr;
2062             }
2063             $descr = undef;
2064             $conf = $res->{snapshots}->{$section} = {};
2065             next;
2066         }
2067
2068         if ($line =~ m/^\#(.*)\s*$/) {
2069             $descr = '' if !defined($descr);
2070             $descr .= PVE::Tools::decode_text($1) . "\n";
2071             next;
2072         }
2073
2074         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2075             $descr = '' if !defined($descr);
2076             $descr .= PVE::Tools::decode_text($2);
2077         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2078             $conf->{snapstate} = $1;
2079         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2080             my $key = $1;
2081             my $value = $2;
2082             $conf->{$key} = $value;
2083         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2084             my $value = $1;
2085             if ($section eq 'pending') {
2086                 $conf->{delete} = $value; # we parse this later
2087             } else {
2088                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2089             }
2090         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2091             my $key = $1;
2092             my $value = $2;
2093             eval { $value = check_type($key, $value); };
2094             if ($@) {
2095                 warn "vm $vmid - unable to parse value of '$key' - $@";
2096             } else {
2097                 $key = 'ide2' if $key eq 'cdrom';
2098                 my $fmt = $confdesc->{$key}->{format};
2099                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2100                     my $v = parse_drive($key, $value);
2101                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2102                         $v->{file} = $volid;
2103                         $value = print_drive($v);
2104                     } else {
2105                         warn "vm $vmid - unable to parse value of '$key'\n";
2106                         next;
2107                     }
2108                 }
2109
2110                 $conf->{$key} = $value;
2111             }
2112         }
2113     }
2114
2115     if (defined($descr)) {
2116         $descr =~ s/\s+$//;
2117         $conf->{description} = $descr;
2118     }
2119     delete $res->{snapstate}; # just to be sure
2120
2121     return $res;
2122 }
2123
2124 sub write_vm_config {
2125     my ($filename, $conf) = @_;
2126
2127     delete $conf->{snapstate}; # just to be sure
2128
2129     if ($conf->{cdrom}) {
2130         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2131         $conf->{ide2} = $conf->{cdrom};
2132         delete $conf->{cdrom};
2133     }
2134
2135     # we do not use 'smp' any longer
2136     if ($conf->{sockets}) {
2137         delete $conf->{smp};
2138     } elsif ($conf->{smp}) {
2139         $conf->{sockets} = $conf->{smp};
2140         delete $conf->{cores};
2141         delete $conf->{smp};
2142     }
2143
2144     my $used_volids = {};
2145
2146     my $cleanup_config = sub {
2147         my ($cref, $pending, $snapname) = @_;
2148
2149         foreach my $key (keys %$cref) {
2150             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2151                 $key eq 'snapstate' || $key eq 'pending';
2152             my $value = $cref->{$key};
2153             if ($key eq 'delete') {
2154                 die "propertry 'delete' is only allowed in [PENDING]\n"
2155                     if !$pending;
2156                 # fixme: check syntax?
2157                 next;
2158             }
2159             eval { $value = check_type($key, $value); };
2160             die "unable to parse value of '$key' - $@" if $@;
2161
2162             $cref->{$key} = $value;
2163
2164             if (!$snapname && is_valid_drivename($key)) {
2165                 my $drive = parse_drive($key, $value);
2166                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2167             }
2168         }
2169     };
2170
2171     &$cleanup_config($conf);
2172
2173     &$cleanup_config($conf->{pending}, 1);
2174
2175     foreach my $snapname (keys %{$conf->{snapshots}}) {
2176         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2177         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2178     }
2179
2180     # remove 'unusedX' settings if we re-add a volume
2181     foreach my $key (keys %$conf) {
2182         my $value = $conf->{$key};
2183         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2184             delete $conf->{$key};
2185         }
2186     }
2187
2188     my $generate_raw_config = sub {
2189         my ($conf, $pending) = @_;
2190
2191         my $raw = '';
2192
2193         # add description as comment to top of file
2194         if (defined(my $descr = $conf->{description})) {
2195             if ($descr) {
2196                 foreach my $cl (split(/\n/, $descr)) {
2197                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2198                 }
2199             } else {
2200                 $raw .= "#\n" if $pending;
2201             }
2202         }
2203
2204         foreach my $key (sort keys %$conf) {
2205             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2206             $raw .= "$key: $conf->{$key}\n";
2207         }
2208         return $raw;
2209     };
2210
2211     my $raw = &$generate_raw_config($conf);
2212
2213     if (scalar(keys %{$conf->{pending}})){
2214         $raw .= "\n[PENDING]\n";
2215         $raw .= &$generate_raw_config($conf->{pending}, 1);
2216     }
2217
2218     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2219         $raw .= "\n[$snapname]\n";
2220         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2221     }
2222
2223     return $raw;
2224 }
2225
2226 sub load_defaults {
2227
2228     my $res = {};
2229
2230     # we use static defaults from our JSON schema configuration
2231     foreach my $key (keys %$confdesc) {
2232         if (defined(my $default = $confdesc->{$key}->{default})) {
2233             $res->{$key} = $default;
2234         }
2235     }
2236
2237     return $res;
2238 }
2239
2240 sub config_list {
2241     my $vmlist = PVE::Cluster::get_vmlist();
2242     my $res = {};
2243     return $res if !$vmlist || !$vmlist->{ids};
2244     my $ids = $vmlist->{ids};
2245     my $nodename = nodename();
2246
2247     foreach my $vmid (keys %$ids) {
2248         my $d = $ids->{$vmid};
2249         next if !$d->{node} || $d->{node} ne $nodename;
2250         next if !$d->{type} || $d->{type} ne 'qemu';
2251         $res->{$vmid}->{exists} = 1;
2252     }
2253     return $res;
2254 }
2255
2256 # test if VM uses local resources (to prevent migration)
2257 sub check_local_resources {
2258     my ($conf, $noerr) = @_;
2259
2260     my @loc_res = ();
2261
2262     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2263     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2264
2265     push @loc_res, "ivshmem" if $conf->{ivshmem};
2266
2267     foreach my $k (keys %$conf) {
2268         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2269         # sockets are safe: they will recreated be on the target side post-migrate
2270         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2271         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2272     }
2273
2274     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2275
2276     return \@loc_res;
2277 }
2278
2279 # check if used storages are available on all nodes (use by migrate)
2280 sub check_storage_availability {
2281     my ($storecfg, $conf, $node) = @_;
2282
2283     PVE::QemuConfig->foreach_volume($conf, sub {
2284         my ($ds, $drive) = @_;
2285
2286         my $volid = $drive->{file};
2287         return if !$volid;
2288
2289         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2290         return if !$sid;
2291
2292         # check if storage is available on both nodes
2293         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2294         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2295    });
2296 }
2297
2298 # list nodes where all VM images are available (used by has_feature API)
2299 sub shared_nodes {
2300     my ($conf, $storecfg) = @_;
2301
2302     my $nodelist = PVE::Cluster::get_nodelist();
2303     my $nodehash = { map { $_ => 1 } @$nodelist };
2304     my $nodename = nodename();
2305
2306     PVE::QemuConfig->foreach_volume($conf, sub {
2307         my ($ds, $drive) = @_;
2308
2309         my $volid = $drive->{file};
2310         return if !$volid;
2311
2312         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2313         if ($storeid) {
2314             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2315             if ($scfg->{disable}) {
2316                 $nodehash = {};
2317             } elsif (my $avail = $scfg->{nodes}) {
2318                 foreach my $node (keys %$nodehash) {
2319                     delete $nodehash->{$node} if !$avail->{$node};
2320                 }
2321             } elsif (!$scfg->{shared}) {
2322                 foreach my $node (keys %$nodehash) {
2323                     delete $nodehash->{$node} if $node ne $nodename
2324                 }
2325             }
2326         }
2327     });
2328
2329     return $nodehash
2330 }
2331
2332 sub check_local_storage_availability {
2333     my ($conf, $storecfg) = @_;
2334
2335     my $nodelist = PVE::Cluster::get_nodelist();
2336     my $nodehash = { map { $_ => {} } @$nodelist };
2337
2338     PVE::QemuConfig->foreach_volume($conf, sub {
2339         my ($ds, $drive) = @_;
2340
2341         my $volid = $drive->{file};
2342         return if !$volid;
2343
2344         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2345         if ($storeid) {
2346             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2347
2348             if ($scfg->{disable}) {
2349                 foreach my $node (keys %$nodehash) {
2350                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2351                 }
2352             } elsif (my $avail = $scfg->{nodes}) {
2353                 foreach my $node (keys %$nodehash) {
2354                     if (!$avail->{$node}) {
2355                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2356                     }
2357                 }
2358             }
2359         }
2360     });
2361
2362     foreach my $node (values %$nodehash) {
2363         if (my $unavail = $node->{unavailable_storages}) {
2364             $node->{unavailable_storages} = [ sort keys %$unavail ];
2365         }
2366     }
2367
2368     return $nodehash
2369 }
2370
2371 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2372 sub check_running {
2373     my ($vmid, $nocheck, $node) = @_;
2374
2375     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2376     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2377 }
2378
2379 sub vzlist {
2380
2381     my $vzlist = config_list();
2382
2383     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2384
2385     while (defined(my $de = $fd->read)) {
2386         next if $de !~ m/^(\d+)\.pid$/;
2387         my $vmid = $1;
2388         next if !defined($vzlist->{$vmid});
2389         if (my $pid = check_running($vmid)) {
2390             $vzlist->{$vmid}->{pid} = $pid;
2391         }
2392     }
2393
2394     return $vzlist;
2395 }
2396
2397 our $vmstatus_return_properties = {
2398     vmid => get_standard_option('pve-vmid'),
2399     status => {
2400         description => "Qemu process status.",
2401         type => 'string',
2402         enum => ['stopped', 'running'],
2403     },
2404     maxmem => {
2405         description => "Maximum memory in bytes.",
2406         type => 'integer',
2407         optional => 1,
2408         renderer => 'bytes',
2409     },
2410     maxdisk => {
2411         description => "Root disk size in bytes.",
2412         type => 'integer',
2413         optional => 1,
2414         renderer => 'bytes',
2415     },
2416     name => {
2417         description => "VM name.",
2418         type => 'string',
2419         optional => 1,
2420     },
2421     qmpstatus => {
2422         description => "Qemu QMP agent status.",
2423         type => 'string',
2424         optional => 1,
2425     },
2426     pid => {
2427         description => "PID of running qemu process.",
2428         type => 'integer',
2429         optional => 1,
2430     },
2431     uptime => {
2432         description => "Uptime.",
2433         type => 'integer',
2434         optional => 1,
2435         renderer => 'duration',
2436     },
2437     cpus => {
2438         description => "Maximum usable CPUs.",
2439         type => 'number',
2440         optional => 1,
2441     },
2442     lock => {
2443         description => "The current config lock, if any.",
2444         type => 'string',
2445         optional => 1,
2446     },
2447     tags => {
2448         description  => "The current configured tags, if any",
2449         type => 'string',
2450         optional => 1,
2451     },
2452 };
2453
2454 my $last_proc_pid_stat;
2455
2456 # get VM status information
2457 # This must be fast and should not block ($full == false)
2458 # We only query KVM using QMP if $full == true (this can be slow)
2459 sub vmstatus {
2460     my ($opt_vmid, $full) = @_;
2461
2462     my $res = {};
2463
2464     my $storecfg = PVE::Storage::config();
2465
2466     my $list = vzlist();
2467     my $defaults = load_defaults();
2468
2469     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2470
2471     my $cpucount = $cpuinfo->{cpus} || 1;
2472
2473     foreach my $vmid (keys %$list) {
2474         next if $opt_vmid && ($vmid ne $opt_vmid);
2475
2476         my $conf = PVE::QemuConfig->load_config($vmid);
2477
2478         my $d = { vmid => $vmid };
2479         $d->{pid} = $list->{$vmid}->{pid};
2480
2481         # fixme: better status?
2482         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2483
2484         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2485         if (defined($size)) {
2486             $d->{disk} = 0; # no info available
2487             $d->{maxdisk} = $size;
2488         } else {
2489             $d->{disk} = 0;
2490             $d->{maxdisk} = 0;
2491         }
2492
2493         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2494             * ($conf->{cores} || $defaults->{cores});
2495         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2496         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2497
2498         $d->{name} = $conf->{name} || "VM $vmid";
2499         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2500             : $defaults->{memory}*(1024*1024);
2501
2502         if ($conf->{balloon}) {
2503             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2504             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2505                 : $defaults->{shares};
2506         }
2507
2508         $d->{uptime} = 0;
2509         $d->{cpu} = 0;
2510         $d->{mem} = 0;
2511
2512         $d->{netout} = 0;
2513         $d->{netin} = 0;
2514
2515         $d->{diskread} = 0;
2516         $d->{diskwrite} = 0;
2517
2518         $d->{template} = PVE::QemuConfig->is_template($conf);
2519
2520         $d->{serial} = 1 if conf_has_serial($conf);
2521         $d->{lock} = $conf->{lock} if $conf->{lock};
2522         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2523
2524         $res->{$vmid} = $d;
2525     }
2526
2527     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2528     foreach my $dev (keys %$netdev) {
2529         next if $dev !~ m/^tap([1-9]\d*)i/;
2530         my $vmid = $1;
2531         my $d = $res->{$vmid};
2532         next if !$d;
2533
2534         $d->{netout} += $netdev->{$dev}->{receive};
2535         $d->{netin} += $netdev->{$dev}->{transmit};
2536
2537         if ($full) {
2538             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2539             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2540         }
2541
2542     }
2543
2544     my $ctime = gettimeofday;
2545
2546     foreach my $vmid (keys %$list) {
2547
2548         my $d = $res->{$vmid};
2549         my $pid = $d->{pid};
2550         next if !$pid;
2551
2552         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2553         next if !$pstat; # not running
2554
2555         my $used = $pstat->{utime} + $pstat->{stime};
2556
2557         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2558
2559         if ($pstat->{vsize}) {
2560             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2561         }
2562
2563         my $old = $last_proc_pid_stat->{$pid};
2564         if (!$old) {
2565             $last_proc_pid_stat->{$pid} = {
2566                 time => $ctime,
2567                 used => $used,
2568                 cpu => 0,
2569             };
2570             next;
2571         }
2572
2573         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2574
2575         if ($dtime > 1000) {
2576             my $dutime = $used -  $old->{used};
2577
2578             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2579             $last_proc_pid_stat->{$pid} = {
2580                 time => $ctime,
2581                 used => $used,
2582                 cpu => $d->{cpu},
2583             };
2584         } else {
2585             $d->{cpu} = $old->{cpu};
2586         }
2587     }
2588
2589     return $res if !$full;
2590
2591     my $qmpclient = PVE::QMPClient->new();
2592
2593     my $ballooncb = sub {
2594         my ($vmid, $resp) = @_;
2595
2596         my $info = $resp->{'return'};
2597         return if !$info->{max_mem};
2598
2599         my $d = $res->{$vmid};
2600
2601         # use memory assigned to VM
2602         $d->{maxmem} = $info->{max_mem};
2603         $d->{balloon} = $info->{actual};
2604
2605         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2606             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2607             $d->{freemem} = $info->{free_mem};
2608         }
2609
2610         $d->{ballooninfo} = $info;
2611     };
2612
2613     my $blockstatscb = sub {
2614         my ($vmid, $resp) = @_;
2615         my $data = $resp->{'return'} || [];
2616         my $totalrdbytes = 0;
2617         my $totalwrbytes = 0;
2618
2619         for my $blockstat (@$data) {
2620             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2621             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2622
2623             $blockstat->{device} =~ s/drive-//;
2624             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2625         }
2626         $res->{$vmid}->{diskread} = $totalrdbytes;
2627         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2628     };
2629
2630     my $statuscb = sub {
2631         my ($vmid, $resp) = @_;
2632
2633         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2634         # this fails if ballon driver is not loaded, so this must be
2635         # the last commnand (following command are aborted if this fails).
2636         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2637
2638         my $status = 'unknown';
2639         if (!defined($status = $resp->{'return'}->{status})) {
2640             warn "unable to get VM status\n";
2641             return;
2642         }
2643
2644         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2645     };
2646
2647     foreach my $vmid (keys %$list) {
2648         next if $opt_vmid && ($vmid ne $opt_vmid);
2649         next if !$res->{$vmid}->{pid}; # not running
2650         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2651     }
2652
2653     $qmpclient->queue_execute(undef, 2);
2654
2655     foreach my $vmid (keys %$list) {
2656         next if $opt_vmid && ($vmid ne $opt_vmid);
2657         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2658     }
2659
2660     return $res;
2661 }
2662
2663 sub conf_has_serial {
2664     my ($conf) = @_;
2665
2666     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2667         if ($conf->{"serial$i"}) {
2668             return 1;
2669         }
2670     }
2671
2672     return 0;
2673 }
2674
2675 sub conf_has_audio {
2676     my ($conf, $id) = @_;
2677
2678     $id //= 0;
2679     my $audio = $conf->{"audio$id"};
2680     return undef if !defined($audio);
2681
2682     my $audioproperties = PVE::JSONSchema::parse_property_string($audio_fmt, $audio);
2683     my $audiodriver = $audioproperties->{driver} // 'spice';
2684
2685     return {
2686         dev => $audioproperties->{device},
2687         dev_id => "audiodev$id",
2688         backend => $audiodriver,
2689         backend_id => "$audiodriver-backend${id}",
2690     };
2691 }
2692
2693 sub audio_devs {
2694     my ($audio, $audiopciaddr, $machine_version) = @_;
2695
2696     my $devs = [];
2697
2698     my $id = $audio->{dev_id};
2699     my $audiodev = "";
2700     if (min_version($machine_version, 4, 2)) {
2701         $audiodev = ",audiodev=$audio->{backend_id}";
2702     }
2703
2704     if ($audio->{dev} eq 'AC97') {
2705         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
2706     } elsif ($audio->{dev} =~ /intel\-hda$/) {
2707         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
2708         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
2709         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
2710     } else {
2711         die "unkown audio device '$audio->{dev}', implement me!";
2712     }
2713
2714     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
2715
2716     return $devs;
2717 }
2718
2719 sub vga_conf_has_spice {
2720     my ($vga) = @_;
2721
2722     my $vgaconf = parse_vga($vga);
2723     my $vgatype = $vgaconf->{type};
2724     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
2725
2726     return $1 || 1;
2727 }
2728
2729 sub is_native($) {
2730     my ($arch) = @_;
2731     return get_host_arch() eq $arch;
2732 }
2733
2734 sub get_vm_arch {
2735     my ($conf) = @_;
2736     return $conf->{arch} // get_host_arch();
2737 }
2738
2739 my $default_machines = {
2740     x86_64 => 'pc',
2741     aarch64 => 'virt',
2742 };
2743
2744 sub get_vm_machine {
2745     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
2746
2747     my $machine = $forcemachine || $conf->{machine};
2748
2749     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
2750         $arch //= 'x86_64';
2751         $machine ||= $default_machines->{$arch};
2752         if ($add_pve_version) {
2753             $kvmversion //= kvm_user_version();
2754             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
2755             $machine .= "+pve$pvever";
2756         }
2757     }
2758
2759     if ($add_pve_version && $machine !~ m/\+pve\d+$/) {
2760         # for version-pinned machines that do not include a pve-version (e.g.
2761         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
2762         $machine .= '+pve0';
2763     }
2764
2765     return $machine;
2766 }
2767
2768 sub get_ovmf_files($) {
2769     my ($arch) = @_;
2770
2771     my $ovmf = $OVMF->{$arch}
2772         or die "no OVMF images known for architecture '$arch'\n";
2773
2774     return @$ovmf;
2775 }
2776
2777 my $Arch2Qemu = {
2778     aarch64 => '/usr/bin/qemu-system-aarch64',
2779     x86_64 => '/usr/bin/qemu-system-x86_64',
2780 };
2781 sub get_command_for_arch($) {
2782     my ($arch) = @_;
2783     return '/usr/bin/kvm' if is_native($arch);
2784
2785     my $cmd = $Arch2Qemu->{$arch}
2786         or die "don't know how to emulate architecture '$arch'\n";
2787     return $cmd;
2788 }
2789
2790 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
2791 # to use in a QEMU command line (-cpu element), first array_intersect the result
2792 # of query_supported_ with query_understood_. This is necessary because:
2793 #
2794 # a) query_understood_ returns flags the host cannot use and
2795 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
2796 #    flags, but CPU settings - with most of them being flags. Those settings
2797 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
2798 #
2799 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
2800 # expensive. If you need the value returned from this, you can get it much
2801 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
2802 # $accel being 'kvm' or 'tcg'.
2803 #
2804 # pvestatd calls this function on startup and whenever the QEMU/KVM version
2805 # changes, automatically populating pmxcfs.
2806 #
2807 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
2808 # since kvm and tcg machines support different flags
2809 #
2810 sub query_supported_cpu_flags {
2811     my ($arch) = @_;
2812
2813     $arch //= get_host_arch();
2814     my $default_machine = $default_machines->{$arch};
2815
2816     my $flags = {};
2817
2818     # FIXME: Once this is merged, the code below should work for ARM as well:
2819     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
2820     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
2821         $arch eq "aarch64";
2822
2823     my $kvm_supported = defined(kvm_version());
2824     my $qemu_cmd = get_command_for_arch($arch);
2825     my $fakevmid = -1;
2826     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
2827
2828     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
2829     my $query_supported_run_qemu = sub {
2830         my ($kvm) = @_;
2831
2832         my $flags = {};
2833         my $cmd = [
2834             $qemu_cmd,
2835             '-machine', $default_machine,
2836             '-display', 'none',
2837             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server,nowait",
2838             '-mon', 'chardev=qmp,mode=control',
2839             '-pidfile', $pidfile,
2840             '-S', '-daemonize'
2841         ];
2842
2843         if (!$kvm) {
2844             push @$cmd, '-accel', 'tcg';
2845         }
2846
2847         my $rc = run_command($cmd, noerr => 1, quiet => 0);
2848         die "QEMU flag querying VM exited with code " . $rc if $rc;
2849
2850         eval {
2851             my $cmd_result = mon_cmd(
2852                 $fakevmid,
2853                 'query-cpu-model-expansion',
2854                 type => 'full',
2855                 model => { name => 'host' }
2856             );
2857
2858             my $props = $cmd_result->{model}->{props};
2859             foreach my $prop (keys %$props) {
2860                 next if $props->{$prop} ne '1';
2861                 # QEMU returns some flags multiple times, with '_', '.' or '-'
2862                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
2863                 # We only keep those with underscores, to match /proc/cpuinfo
2864                 $prop =~ s/\.|-/_/g;
2865                 $flags->{$prop} = 1;
2866             }
2867         };
2868         my $err = $@;
2869
2870         # force stop with 10 sec timeout and 'nocheck'
2871         # always stop, even if QMP failed
2872         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
2873
2874         die $err if $err;
2875
2876         return [ sort keys %$flags ];
2877     };
2878
2879     # We need to query QEMU twice, since KVM and TCG have different supported flags
2880     PVE::QemuConfig->lock_config($fakevmid, sub {
2881         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
2882         warn "warning: failed querying supported tcg flags: $@\n" if $@;
2883
2884         if ($kvm_supported) {
2885             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
2886             warn "warning: failed querying supported kvm flags: $@\n" if $@;
2887         }
2888     });
2889
2890     return $flags;
2891 }
2892
2893 # Understood CPU flags are written to a file at 'pve-qemu' compile time
2894 my $understood_cpu_flag_dir = "/usr/share/kvm";
2895 sub query_understood_cpu_flags {
2896     my $arch = get_host_arch();
2897     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
2898
2899     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
2900         if ! -e $filepath;
2901
2902     my $raw = file_get_contents($filepath);
2903     $raw =~ s/^\s+|\s+$//g;
2904     my @flags = split(/\s+/, $raw);
2905
2906     return \@flags;
2907 }
2908
2909 sub config_to_command {
2910     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu) = @_;
2911
2912     my $cmd = [];
2913     my $globalFlags = [];
2914     my $machineFlags = [];
2915     my $rtcFlags = [];
2916     my $devices = [];
2917     my $pciaddr = '';
2918     my $bridges = {};
2919     my $ostype = $conf->{ostype};
2920     my $winversion = windows_version($ostype);
2921     my $kvm = $conf->{kvm};
2922     my $nodename = nodename();
2923
2924     my $arch = get_vm_arch($conf);
2925     my $kvm_binary = get_command_for_arch($arch);
2926     my $kvmver = kvm_user_version($kvm_binary);
2927
2928     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
2929         $kvmver //= "undefined";
2930         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
2931     }
2932
2933     my $add_pve_version = min_version($kvmver, 4, 1);
2934
2935     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
2936     my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type, $kvmver);
2937     $kvm //= 1 if is_native($arch);
2938
2939     $machine_version =~ m/(\d+)\.(\d+)/;
2940     my ($machine_major, $machine_minor) = ($1, $2);
2941
2942     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
2943         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
2944     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
2945         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type', please upgrade node '$nodename'\n"
2946     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
2947         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
2948         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is pve$max_pve_version)"
2949             ." is too old to run machine type '$machine_type', please upgrade node '$nodename'\n";
2950     }
2951
2952     # if a specific +pve version is required for a feature, use $version_guard
2953     # instead of min_version to allow machines to be run with the minimum
2954     # required version
2955     my $required_pve_version = 0;
2956     my $version_guard = sub {
2957         my ($major, $minor, $pve) = @_;
2958         return 0 if !min_version($machine_version, $major, $minor, $pve);
2959         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
2960         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
2961         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
2962         return 1;
2963     };
2964
2965     if ($kvm) {
2966         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
2967             if !defined kvm_version();
2968     }
2969
2970     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2971     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2972     my $use_old_bios_files = undef;
2973     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2974
2975     my $cpuunits = defined($conf->{cpuunits}) ?
2976             $conf->{cpuunits} : $defaults->{cpuunits};
2977
2978     push @$cmd, $kvm_binary;
2979
2980     push @$cmd, '-id', $vmid;
2981
2982     my $vmname = $conf->{name} || "vm$vmid";
2983
2984     push @$cmd, '-name', $vmname;
2985
2986     my $use_virtio = 0;
2987
2988     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
2989     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2990     push @$cmd, '-mon', "chardev=qmp,mode=control";
2991
2992     if (min_version($machine_version, 2, 12)) {
2993         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
2994         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
2995     }
2996
2997     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
2998
2999     push @$cmd, '-daemonize';
3000
3001     if ($conf->{smbios1}) {
3002         my $smbios_conf = parse_smbios1($conf->{smbios1});
3003         if ($smbios_conf->{base64}) {
3004             # Do not pass base64 flag to qemu
3005             delete $smbios_conf->{base64};
3006             my $smbios_string = "";
3007             foreach my $key (keys %$smbios_conf) {
3008                 my $value;
3009                 if ($key eq "uuid") {
3010                     $value = $smbios_conf->{uuid}
3011                 } else {
3012                     $value = decode_base64($smbios_conf->{$key});
3013                 }
3014                 # qemu accepts any binary data, only commas need escaping by double comma
3015                 $value =~ s/,/,,/g;
3016                 $smbios_string .= "," . $key . "=" . $value if $value;
3017             }
3018             push @$cmd, '-smbios', "type=1" . $smbios_string;
3019         } else {
3020             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3021         }
3022     }
3023
3024     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3025         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3026         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3027
3028         my ($path, $format);
3029         if (my $efidisk = $conf->{efidisk0}) {
3030             my $d = parse_drive('efidisk0', $efidisk);
3031             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3032             $format = $d->{format};
3033             if ($storeid) {
3034                 $path = PVE::Storage::path($storecfg, $d->{file});
3035                 if (!defined($format)) {
3036                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3037                     $format = qemu_img_format($scfg, $volname);
3038                 }
3039             } else {
3040                 $path = $d->{file};
3041                 die "efidisk format must be specified\n"
3042                     if !defined($format);
3043             }
3044         } else {
3045             warn "no efidisk configured! Using temporary efivars disk.\n";
3046             $path = "/tmp/$vmid-ovmf.fd";
3047             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3048             $format = 'raw';
3049         }
3050
3051         my $size_str = "";
3052
3053         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3054             $size_str = ",size=" . (-s $ovmf_vars);
3055         }
3056
3057         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3058         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0$size_str,file=$path";
3059     }
3060
3061     # load q35 config
3062     if ($q35) {
3063         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3064         if (min_version($machine_version, 4, 0)) {
3065             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3066         } else {
3067             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3068         }
3069     }
3070
3071     if ($conf->{vmgenid}) {
3072         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3073     }
3074
3075     # add usb controllers
3076     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3077     push @$devices, @usbcontrollers if @usbcontrollers;
3078     my $vga = parse_vga($conf->{vga});
3079
3080     my $qxlnum = vga_conf_has_spice($conf->{vga});
3081     $vga->{type} = 'qxl' if $qxlnum;
3082
3083     if (!$vga->{type}) {
3084         if ($arch eq 'aarch64') {
3085             $vga->{type} = 'virtio';
3086         } elsif (min_version($machine_version, 2, 9)) {
3087             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3088         } else {
3089             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3090         }
3091     }
3092
3093     # enable absolute mouse coordinates (needed by vnc)
3094     my $tablet;
3095     if (defined($conf->{tablet})) {
3096         $tablet = $conf->{tablet};
3097     } else {
3098         $tablet = $defaults->{tablet};
3099         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3100         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3101     }
3102
3103     if ($tablet) {
3104         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3105         my $kbd = print_keyboarddevice_full($conf, $arch);
3106         push @$devices, '-device', $kbd if defined($kbd);
3107     }
3108
3109     # host pci device passthrough
3110     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3111         $vmid, $conf, $devices, $winversion, $q35, $bridges, $arch, $machine_type);
3112
3113     # usb devices
3114     my $usb_dev_features = {};
3115     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3116
3117     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features);
3118     push @$devices, @usbdevices if @usbdevices;
3119     # serial devices
3120     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3121         if (my $path = $conf->{"serial$i"}) {
3122             if ($path eq 'socket') {
3123                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3124                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3125                 # On aarch64, serial0 is the UART device. Qemu only allows
3126                 # connecting UART devices via the '-serial' command line, as
3127                 # the device has a fixed slot on the hardware...
3128                 if ($arch eq 'aarch64' && $i == 0) {
3129                     push @$devices, '-serial', "chardev:serial$i";
3130                 } else {
3131                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3132                 }
3133             } else {
3134                 die "no such serial device\n" if ! -c $path;
3135                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3136                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3137             }
3138         }
3139     }
3140
3141     # parallel devices
3142     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3143         if (my $path = $conf->{"parallel$i"}) {
3144             die "no such parallel device\n" if ! -c $path;
3145             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3146             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3147             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3148         }
3149     }
3150
3151     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3152         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3153         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3154         push @$devices, @$audio_devs;
3155     }
3156
3157     my $sockets = 1;
3158     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3159     $sockets = $conf->{sockets} if  $conf->{sockets};
3160
3161     my $cores = $conf->{cores} || 1;
3162
3163     my $maxcpus = $sockets * $cores;
3164
3165     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3166
3167     my $allowed_vcpus = $cpuinfo->{cpus};
3168
3169     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3170         if ($allowed_vcpus < $maxcpus);
3171
3172     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3173
3174         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3175         for (my $i = 2; $i <= $vcpus; $i++)  {
3176             my $cpustr = print_cpu_device($conf,$i);
3177             push @$cmd, '-device', $cpustr;
3178         }
3179
3180     } else {
3181
3182         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3183     }
3184     push @$cmd, '-nodefaults';
3185
3186     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3187
3188     my $bootindex_hash = {};
3189     my $i = 1;
3190     foreach my $o (split(//, $bootorder)) {
3191         $bootindex_hash->{$o} = $i*100;
3192         $i++;
3193     }
3194
3195     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3196
3197     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3198
3199     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3200
3201     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3202         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3203         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3204         push @$cmd,  '-vnc', "unix:$socket,password";
3205     } else {
3206         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3207         push @$cmd, '-nographic';
3208     }
3209
3210     # time drift fix
3211     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3212     my $useLocaltime = $conf->{localtime};
3213
3214     if ($winversion >= 5) { # windows
3215         $useLocaltime = 1 if !defined($conf->{localtime});
3216
3217         # use time drift fix when acpi is enabled
3218         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3219             $tdf = 1 if !defined($conf->{tdf});
3220         }
3221     }
3222
3223     if ($winversion >= 6) {
3224         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3225         push @$cmd, '-no-hpet';
3226     }
3227
3228     push @$rtcFlags, 'driftfix=slew' if $tdf;
3229
3230     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3231         push @$rtcFlags, "base=$conf->{startdate}";
3232     } elsif ($useLocaltime) {
3233         push @$rtcFlags, 'base=localtime';
3234     }
3235
3236     if ($forcecpu) {
3237         push @$cmd, '-cpu', $forcecpu;
3238     } else {
3239         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3240     }
3241
3242     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3243
3244     push @$cmd, '-S' if $conf->{freeze};
3245
3246     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3247
3248     my $guest_agent = parse_guest_agent($conf);
3249
3250     if ($guest_agent->{enabled}) {
3251         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3252         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3253
3254         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3255             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3256             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3257             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3258         } elsif ($guest_agent->{type} eq 'isa') {
3259             push @$devices, '-device', "isa-serial,chardev=qga0";
3260         }
3261     }
3262
3263     my $rng = parse_rng($conf->{rng0}) if $conf->{rng0};
3264     if ($rng && &$version_guard(4, 1, 2)) {
3265         check_rng_source($rng->{source});
3266
3267         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3268         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3269         my $limiter_str = "";
3270         if ($max_bytes) {
3271             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3272         }
3273
3274         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3275         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3276         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3277     }
3278
3279     my $spice_port;
3280
3281     if ($qxlnum) {
3282         if ($qxlnum > 1) {
3283             if ($winversion){
3284                 for (my $i = 1; $i < $qxlnum; $i++){
3285                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3286                 }
3287             } else {
3288                 # assume other OS works like Linux
3289                 my ($ram, $vram) = ("134217728", "67108864");
3290                 if ($vga->{memory}) {
3291                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3292                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3293                 }
3294                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3295                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3296             }
3297         }
3298
3299         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3300
3301         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3302         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3303         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3304
3305         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3306         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3307         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3308
3309         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3310         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3311
3312         my $spice_enhancement = PVE::JSONSchema::parse_property_string($spice_enhancements_fmt, $conf->{spice_enhancements} // '');
3313         if ($spice_enhancement->{foldersharing}) {
3314             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3315             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3316         }
3317
3318         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3319         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming};
3320         push @$devices, '-spice', "$spice_opts";
3321     }
3322
3323     # enable balloon by default, unless explicitly disabled
3324     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3325         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3326         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3327     }
3328
3329     if ($conf->{watchdog}) {
3330         my $wdopts = parse_watchdog($conf->{watchdog});
3331         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3332         my $watchdog = $wdopts->{model} || 'i6300esb';
3333         push @$devices, '-device', "$watchdog$pciaddr";
3334         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3335     }
3336
3337     my $vollist = [];
3338     my $scsicontroller = {};
3339     my $ahcicontroller = {};
3340     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3341
3342     # Add iscsi initiator name if available
3343     if (my $initiator = get_initiator_name()) {
3344         push @$devices, '-iscsi', "initiator-name=$initiator";
3345     }
3346
3347     PVE::QemuConfig->foreach_volume($conf, sub {
3348         my ($ds, $drive) = @_;
3349
3350         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3351             push @$vollist, $drive->{file};
3352         }
3353
3354         # ignore efidisk here, already added in bios/fw handling code above
3355         return if $drive->{interface} eq 'efidisk';
3356
3357         $use_virtio = 1 if $ds =~ m/^virtio/;
3358
3359         if (drive_is_cdrom ($drive)) {
3360             if ($bootindex_hash->{d}) {
3361                 $drive->{bootindex} = $bootindex_hash->{d};
3362                 $bootindex_hash->{d} += 1;
3363             }
3364         } else {
3365             if ($bootindex_hash->{c}) {
3366                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3367                 $bootindex_hash->{c} += 1;
3368             }
3369         }
3370
3371         if ($drive->{interface} eq 'virtio'){
3372            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3373         }
3374
3375         if ($drive->{interface} eq 'scsi') {
3376
3377             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3378
3379             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3380                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3381
3382             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3383             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3384
3385             my $iothread = '';
3386             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3387                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3388                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3389             } elsif ($drive->{iothread}) {
3390                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3391             }
3392
3393             my $queues = '';
3394             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3395                 $queues = ",num_queues=$drive->{queues}";
3396             }
3397
3398             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3399             $scsicontroller->{$controller}=1;
3400         }
3401
3402         if ($drive->{interface} eq 'sata') {
3403             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3404             $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3405             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3406             $ahcicontroller->{$controller}=1;
3407         }
3408
3409         my $drive_cmd = print_drive_commandline_full($storecfg, $vmid, $drive);
3410         $drive_cmd .= ',readonly' if PVE::QemuConfig->is_template($conf);
3411
3412         push @$devices, '-drive',$drive_cmd;
3413         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3414     });
3415
3416     for (my $i = 0; $i < $MAX_NETS; $i++) {
3417          next if !$conf->{"net$i"};
3418          my $d = parse_net($conf->{"net$i"});
3419          next if !$d;
3420
3421          $use_virtio = 1 if $d->{model} eq 'virtio';
3422
3423          if ($bootindex_hash->{n}) {
3424             $d->{bootindex} = $bootindex_hash->{n};
3425             $bootindex_hash->{n} += 1;
3426          }
3427
3428          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
3429          push @$devices, '-netdev', $netdevfull;
3430
3431          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
3432          push @$devices, '-device', $netdevicefull;
3433     }
3434
3435     if ($conf->{ivshmem}) {
3436         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3437
3438         my $bus;
3439         if ($q35) {
3440             $bus = print_pcie_addr("ivshmem");
3441         } else {
3442             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3443         }
3444
3445         my $ivshmem_name = $ivshmem->{name} // $vmid;
3446         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3447
3448         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3449         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
3450     }
3451
3452     # pci.4 is nested in pci.1
3453     $bridges->{1} = 1 if $bridges->{4};
3454
3455     if (!$q35) {
3456         # add pci bridges
3457         if (min_version($machine_version, 2, 3)) {
3458            $bridges->{1} = 1;
3459            $bridges->{2} = 1;
3460         }
3461
3462         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3463
3464     }
3465
3466     for my $k (sort {$b cmp $a} keys %$bridges) {
3467         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3468
3469         my $k_name = $k;
3470         if ($k == 2 && $legacy_igd) {
3471             $k_name = "$k-igd";
3472         }
3473         $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
3474
3475         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3476         if ($q35) {
3477             # add after -readconfig pve-q35.cfg
3478             splice @$devices, 2, 0, '-device', $devstr;
3479         } else {
3480             unshift @$devices, '-device', $devstr if $k > 0;
3481         }
3482     }
3483
3484     if (!$kvm) {
3485         push @$machineFlags, 'accel=tcg';
3486     }
3487
3488     my $machine_type_min = $machine_type;
3489     if ($add_pve_version) {
3490         $machine_type_min =~ s/\+pve\d+$//;
3491         $machine_type_min .= "+pve$required_pve_version";
3492     }
3493     push @$machineFlags, "type=${machine_type_min}";
3494
3495     push @$cmd, @$devices;
3496     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
3497     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
3498     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
3499
3500     if (my $vmstate = $conf->{vmstate}) {
3501         my $statepath = PVE::Storage::path($storecfg, $vmstate);
3502         push @$vollist, $vmstate;
3503         push @$cmd, '-loadstate', $statepath;
3504         print "activating and using '$vmstate' as vmstate\n";
3505     }
3506
3507     # add custom args
3508     if ($conf->{args}) {
3509         my $aa = PVE::Tools::split_args($conf->{args});
3510         push @$cmd, @$aa;
3511     }
3512
3513     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3514 }
3515
3516 sub check_rng_source {
3517     my ($source) = @_;
3518
3519     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3520     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
3521         if ! -e $source;
3522
3523     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
3524     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
3525         # Needs to abort, otherwise QEMU crashes on first rng access.
3526         # Note that rng_current cannot be changed to 'none' manually, so
3527         # once the VM is past this point, it is no longer an issue.
3528         die "Cannot start VM with passed-through RNG device: '/dev/hwrng'"
3529             . " exists, but '$rng_current' is set to 'none'. Ensure that"
3530             . " a compatible hardware-RNG is attached to the host.\n";
3531     }
3532 }
3533
3534 sub spice_port {
3535     my ($vmid) = @_;
3536
3537     my $res = mon_cmd($vmid, 'query-spice');
3538
3539     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3540 }
3541
3542 sub vm_devices_list {
3543     my ($vmid) = @_;
3544
3545     my $res = mon_cmd($vmid, 'query-pci');
3546     my $devices_to_check = [];
3547     my $devices = {};
3548     foreach my $pcibus (@$res) {
3549         push @$devices_to_check, @{$pcibus->{devices}},
3550     }
3551
3552     while (@$devices_to_check) {
3553         my $to_check = [];
3554         for my $d (@$devices_to_check) {
3555             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3556             next if !$d->{'pci_bridge'};
3557
3558             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3559             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3560         }
3561         $devices_to_check = $to_check;
3562     }
3563
3564     my $resblock = mon_cmd($vmid, 'query-block');
3565     foreach my $block (@$resblock) {
3566         if($block->{device} =~ m/^drive-(\S+)/){
3567                 $devices->{$1} = 1;
3568         }
3569     }
3570
3571     my $resmice = mon_cmd($vmid, 'query-mice');
3572     foreach my $mice (@$resmice) {
3573         if ($mice->{name} eq 'QEMU HID Tablet') {
3574             $devices->{tablet} = 1;
3575             last;
3576         }
3577     }
3578
3579     # for usb devices there is no query-usb
3580     # but we can iterate over the entries in
3581     # qom-list path=/machine/peripheral
3582     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3583     foreach my $per (@$resperipheral) {
3584         if ($per->{name} =~ m/^usb\d+$/) {
3585             $devices->{$per->{name}} = 1;
3586         }
3587     }
3588
3589     return $devices;
3590 }
3591
3592 sub vm_deviceplug {
3593     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3594
3595     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3596
3597     my $devices_list = vm_devices_list($vmid);
3598     return 1 if defined($devices_list->{$deviceid});
3599
3600     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
3601
3602     if ($deviceid eq 'tablet') {
3603
3604         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
3605
3606     } elsif ($deviceid eq 'keyboard') {
3607
3608         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
3609
3610     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3611
3612         die "usb hotplug currently not reliable\n";
3613         # since we can't reliably hot unplug all added usb devices
3614         # and usb passthrough disables live migration
3615         # we disable usb hotplugging for now
3616         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3617
3618     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3619
3620         qemu_iothread_add($vmid, $deviceid, $device);
3621
3622         qemu_driveadd($storecfg, $vmid, $device);
3623         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
3624
3625         qemu_deviceadd($vmid, $devicefull);
3626         eval { qemu_deviceaddverify($vmid, $deviceid); };
3627         if (my $err = $@) {
3628             eval { qemu_drivedel($vmid, $deviceid); };
3629             warn $@ if $@;
3630             die $err;
3631         }
3632
3633     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3634
3635
3636         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3637         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3638         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3639
3640         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3641
3642         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3643             qemu_iothread_add($vmid, $deviceid, $device);
3644             $devicefull .= ",iothread=iothread-$deviceid";
3645         }
3646
3647         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3648             $devicefull .= ",num_queues=$device->{queues}";
3649         }
3650
3651         qemu_deviceadd($vmid, $devicefull);
3652         qemu_deviceaddverify($vmid, $deviceid);
3653
3654     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3655
3656         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
3657         qemu_driveadd($storecfg, $vmid, $device);
3658
3659         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
3660         eval { qemu_deviceadd($vmid, $devicefull); };
3661         if (my $err = $@) {
3662             eval { qemu_drivedel($vmid, $deviceid); };
3663             warn $@ if $@;
3664             die $err;
3665         }
3666
3667     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3668
3669         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
3670
3671         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
3672         my $use_old_bios_files = undef;
3673         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3674
3675         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
3676         qemu_deviceadd($vmid, $netdevicefull);
3677         eval {
3678             qemu_deviceaddverify($vmid, $deviceid);
3679             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
3680         };
3681         if (my $err = $@) {
3682             eval { qemu_netdevdel($vmid, $deviceid); };
3683             warn $@ if $@;
3684             die $err;
3685         }
3686
3687     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3688
3689         my $bridgeid = $2;
3690         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3691         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3692
3693         qemu_deviceadd($vmid, $devicefull);
3694         qemu_deviceaddverify($vmid, $deviceid);
3695
3696     } else {
3697         die "can't hotplug device '$deviceid'\n";
3698     }
3699
3700     return 1;
3701 }
3702
3703 # fixme: this should raise exceptions on error!
3704 sub vm_deviceunplug {
3705     my ($vmid, $conf, $deviceid) = @_;
3706
3707     my $devices_list = vm_devices_list($vmid);
3708     return 1 if !defined($devices_list->{$deviceid});
3709
3710     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3711
3712     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
3713
3714         qemu_devicedel($vmid, $deviceid);
3715
3716     } elsif ($deviceid =~ m/^usb\d+$/) {
3717
3718         die "usb hotplug currently not reliable\n";
3719         # when unplugging usb devices this way,
3720         # there may be remaining usb controllers/hubs
3721         # so we disable it for now
3722         qemu_devicedel($vmid, $deviceid);
3723         qemu_devicedelverify($vmid, $deviceid);
3724
3725     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3726
3727         qemu_devicedel($vmid, $deviceid);
3728         qemu_devicedelverify($vmid, $deviceid);
3729         qemu_drivedel($vmid, $deviceid);
3730         qemu_iothread_del($conf, $vmid, $deviceid);
3731
3732     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3733
3734         qemu_devicedel($vmid, $deviceid);
3735         qemu_devicedelverify($vmid, $deviceid);
3736         qemu_iothread_del($conf, $vmid, $deviceid);
3737
3738     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3739
3740         qemu_devicedel($vmid, $deviceid);
3741         qemu_drivedel($vmid, $deviceid);
3742         qemu_deletescsihw($conf, $vmid, $deviceid);
3743
3744     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3745
3746         qemu_devicedel($vmid, $deviceid);
3747         qemu_devicedelverify($vmid, $deviceid);
3748         qemu_netdevdel($vmid, $deviceid);
3749
3750     } else {
3751         die "can't unplug device '$deviceid'\n";
3752     }
3753
3754     return 1;
3755 }
3756
3757 sub qemu_deviceadd {
3758     my ($vmid, $devicefull) = @_;
3759
3760     $devicefull = "driver=".$devicefull;
3761     my %options =  split(/[=,]/, $devicefull);
3762
3763     mon_cmd($vmid, "device_add" , %options);
3764 }
3765
3766 sub qemu_devicedel {
3767     my ($vmid, $deviceid) = @_;
3768
3769     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
3770 }
3771
3772 sub qemu_iothread_add {
3773     my($vmid, $deviceid, $device) = @_;
3774
3775     if ($device->{iothread}) {
3776         my $iothreads = vm_iothreads_list($vmid);
3777         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3778     }
3779 }
3780
3781 sub qemu_iothread_del {
3782     my($conf, $vmid, $deviceid) = @_;
3783
3784     my $confid = $deviceid;
3785     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
3786         $confid = 'scsi' . $1;
3787     }
3788     my $device = parse_drive($confid, $conf->{$confid});
3789     if ($device->{iothread}) {
3790         my $iothreads = vm_iothreads_list($vmid);
3791         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3792     }
3793 }
3794
3795 sub qemu_objectadd {
3796     my($vmid, $objectid, $qomtype) = @_;
3797
3798     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3799
3800     return 1;
3801 }
3802
3803 sub qemu_objectdel {
3804     my($vmid, $objectid) = @_;
3805
3806     mon_cmd($vmid, "object-del", id => $objectid);
3807
3808     return 1;
3809 }
3810
3811 sub qemu_driveadd {
3812     my ($storecfg, $vmid, $device) = @_;
3813
3814     my $drive = print_drive_commandline_full($storecfg, $vmid, $device);
3815     $drive =~ s/\\/\\\\/g;
3816     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
3817
3818     # If the command succeeds qemu prints: "OK"
3819     return 1 if $ret =~ m/OK/s;
3820
3821     die "adding drive failed: $ret\n";
3822 }
3823
3824 sub qemu_drivedel {
3825     my($vmid, $deviceid) = @_;
3826
3827     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
3828     $ret =~ s/^\s+//;
3829
3830     return 1 if $ret eq "";
3831
3832     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3833     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3834
3835     die "deleting drive $deviceid failed : $ret\n";
3836 }
3837
3838 sub qemu_deviceaddverify {
3839     my ($vmid, $deviceid) = @_;
3840
3841     for (my $i = 0; $i <= 5; $i++) {
3842          my $devices_list = vm_devices_list($vmid);
3843          return 1 if defined($devices_list->{$deviceid});
3844          sleep 1;
3845     }
3846
3847     die "error on hotplug device '$deviceid'\n";
3848 }
3849
3850
3851 sub qemu_devicedelverify {
3852     my ($vmid, $deviceid) = @_;
3853
3854     # need to verify that the device is correctly removed as device_del
3855     # is async and empty return is not reliable
3856
3857     for (my $i = 0; $i <= 5; $i++) {
3858          my $devices_list = vm_devices_list($vmid);
3859          return 1 if !defined($devices_list->{$deviceid});
3860          sleep 1;
3861     }
3862
3863     die "error on hot-unplugging device '$deviceid'\n";
3864 }
3865
3866 sub qemu_findorcreatescsihw {
3867     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
3868
3869     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3870
3871     my $scsihwid="$controller_prefix$controller";
3872     my $devices_list = vm_devices_list($vmid);
3873
3874     if(!defined($devices_list->{$scsihwid})) {
3875         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
3876     }
3877
3878     return 1;
3879 }
3880
3881 sub qemu_deletescsihw {
3882     my ($conf, $vmid, $opt) = @_;
3883
3884     my $device = parse_drive($opt, $conf->{$opt});
3885
3886     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3887         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3888         return 1;
3889     }
3890
3891     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3892
3893     my $devices_list = vm_devices_list($vmid);
3894     foreach my $opt (keys %{$devices_list}) {
3895         if (is_valid_drivename($opt)) {
3896             my $drive = parse_drive($opt, $conf->{$opt});
3897             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3898                 return 1;
3899             }
3900         }
3901     }
3902
3903     my $scsihwid="scsihw$controller";
3904
3905     vm_deviceunplug($vmid, $conf, $scsihwid);
3906
3907     return 1;
3908 }
3909
3910 sub qemu_add_pci_bridge {
3911     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
3912
3913     my $bridges = {};
3914
3915     my $bridgeid;
3916
3917     print_pci_addr($device, $bridges, $arch, $machine_type);
3918
3919     while (my ($k, $v) = each %$bridges) {
3920         $bridgeid = $k;
3921     }
3922     return 1 if !defined($bridgeid) || $bridgeid < 1;
3923
3924     my $bridge = "pci.$bridgeid";
3925     my $devices_list = vm_devices_list($vmid);
3926
3927     if (!defined($devices_list->{$bridge})) {
3928         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
3929     }
3930
3931     return 1;
3932 }
3933
3934 sub qemu_set_link_status {
3935     my ($vmid, $device, $up) = @_;
3936
3937     mon_cmd($vmid, "set_link", name => $device,
3938                up => $up ? JSON::true : JSON::false);
3939 }
3940
3941 sub qemu_netdevadd {
3942     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
3943
3944     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
3945     my %options =  split(/[=,]/, $netdev);
3946
3947     if (defined(my $vhost = $options{vhost})) {
3948         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
3949     }
3950
3951     if (defined(my $queues = $options{queues})) {
3952         $options{queues} = $queues + 0;
3953     }
3954
3955     mon_cmd($vmid, "netdev_add",  %options);
3956     return 1;
3957 }
3958
3959 sub qemu_netdevdel {
3960     my ($vmid, $deviceid) = @_;
3961
3962     mon_cmd($vmid, "netdev_del", id => $deviceid);
3963 }
3964
3965 sub qemu_usb_hotplug {
3966     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3967
3968     return if !$device;
3969
3970     # remove the old one first
3971     vm_deviceunplug($vmid, $conf, $deviceid);
3972
3973     # check if xhci controller is necessary and available
3974     if ($device->{usb3}) {
3975
3976         my $devicelist = vm_devices_list($vmid);
3977
3978         if (!$devicelist->{xhci}) {
3979             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
3980             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
3981         }
3982     }
3983     my $d = parse_usb_device($device->{host});
3984     $d->{usb3} = $device->{usb3};
3985
3986     # add the new one
3987     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
3988 }
3989
3990 sub qemu_cpu_hotplug {
3991     my ($vmid, $conf, $vcpus) = @_;
3992
3993     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
3994
3995     my $sockets = 1;
3996     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3997     $sockets = $conf->{sockets} if  $conf->{sockets};
3998     my $cores = $conf->{cores} || 1;
3999     my $maxcpus = $sockets * $cores;
4000
4001     $vcpus = $maxcpus if !$vcpus;
4002
4003     die "you can't add more vcpus than maxcpus\n"
4004         if $vcpus > $maxcpus;
4005
4006     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4007
4008     if ($vcpus < $currentvcpus) {
4009
4010         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4011
4012             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4013                 qemu_devicedel($vmid, "cpu$i");
4014                 my $retry = 0;
4015                 my $currentrunningvcpus = undef;
4016                 while (1) {
4017                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4018                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4019                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4020                     $retry++;
4021                     sleep 1;
4022                 }
4023                 #update conf after each succesfull cpu unplug
4024                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4025                 PVE::QemuConfig->write_config($vmid, $conf);
4026             }
4027         } else {
4028             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4029         }
4030
4031         return;
4032     }
4033
4034     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4035     die "vcpus in running vm does not match its configuration\n"
4036         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4037
4038     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4039
4040         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4041             my $cpustr = print_cpu_device($conf, $i);
4042             qemu_deviceadd($vmid, $cpustr);
4043
4044             my $retry = 0;
4045             my $currentrunningvcpus = undef;
4046             while (1) {
4047                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4048                 last if scalar(@{$currentrunningvcpus}) == $i;
4049                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4050                 sleep 1;
4051                 $retry++;
4052             }
4053             #update conf after each succesfull cpu hotplug
4054             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4055             PVE::QemuConfig->write_config($vmid, $conf);
4056         }
4057     } else {
4058
4059         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4060             mon_cmd($vmid, "cpu-add", id => int($i));
4061         }
4062     }
4063 }
4064
4065 sub qemu_block_set_io_throttle {
4066     my ($vmid, $deviceid,
4067         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4068         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4069         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4070         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4071
4072     return if !check_running($vmid) ;
4073
4074     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4075         bps => int($bps),
4076         bps_rd => int($bps_rd),
4077         bps_wr => int($bps_wr),
4078         iops => int($iops),
4079         iops_rd => int($iops_rd),
4080         iops_wr => int($iops_wr),
4081         bps_max => int($bps_max),
4082         bps_rd_max => int($bps_rd_max),
4083         bps_wr_max => int($bps_wr_max),
4084         iops_max => int($iops_max),
4085         iops_rd_max => int($iops_rd_max),
4086         iops_wr_max => int($iops_wr_max),
4087         bps_max_length => int($bps_max_length),
4088         bps_rd_max_length => int($bps_rd_max_length),
4089         bps_wr_max_length => int($bps_wr_max_length),
4090         iops_max_length => int($iops_max_length),
4091         iops_rd_max_length => int($iops_rd_max_length),
4092         iops_wr_max_length => int($iops_wr_max_length),
4093     );
4094
4095 }
4096
4097 # old code, only used to shutdown old VM after update
4098 sub __read_avail {
4099     my ($fh, $timeout) = @_;
4100
4101     my $sel = new IO::Select;
4102     $sel->add($fh);
4103
4104     my $res = '';
4105     my $buf;
4106
4107     my @ready;
4108     while (scalar (@ready = $sel->can_read($timeout))) {
4109         my $count;
4110         if ($count = $fh->sysread($buf, 8192)) {
4111             if ($buf =~ /^(.*)\(qemu\) $/s) {
4112                 $res .= $1;
4113                 last;
4114             } else {
4115                 $res .= $buf;
4116             }
4117         } else {
4118             if (!defined($count)) {
4119                 die "$!\n";
4120             }
4121             last;
4122         }
4123     }
4124
4125     die "monitor read timeout\n" if !scalar(@ready);
4126
4127     return $res;
4128 }
4129
4130 sub qemu_block_resize {
4131     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4132
4133     my $running = check_running($vmid);
4134
4135     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4136
4137     return if !$running;
4138
4139     my $padding = (1024 - $size % 1024) % 1024;
4140     $size = $size + $padding;
4141
4142     mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4143
4144 }
4145
4146 sub qemu_volume_snapshot {
4147     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4148
4149     my $running = check_running($vmid);
4150
4151     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4152         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4153     } else {
4154         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4155     }
4156 }
4157
4158 sub qemu_volume_snapshot_delete {
4159     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4160
4161     my $running = check_running($vmid);
4162
4163     if($running) {
4164
4165         $running = undef;
4166         my $conf = PVE::QemuConfig->load_config($vmid);
4167         PVE::QemuConfig->foreach_volume($conf, sub {
4168             my ($ds, $drive) = @_;
4169             $running = 1 if $drive->{file} eq $volid;
4170         });
4171     }
4172
4173     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4174         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4175     } else {
4176         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4177     }
4178 }
4179
4180 sub set_migration_caps {
4181     my ($vmid) = @_;
4182
4183     my $cap_ref = [];
4184
4185     my $enabled_cap = {
4186         "auto-converge" => 1,
4187         "xbzrle" => 1,
4188         "x-rdma-pin-all" => 0,
4189         "zero-blocks" => 0,
4190         "compress" => 0
4191     };
4192
4193     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4194
4195     for my $supported_capability (@$supported_capabilities) {
4196         push @$cap_ref, {
4197             capability => $supported_capability->{capability},
4198             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4199         };
4200     }
4201
4202     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4203 }
4204
4205 sub foreach_volid {
4206     my ($conf, $func, @param) = @_;
4207
4208     my $volhash = {};
4209
4210     my $test_volid = sub {
4211         my ($key, $drive, $snapname) = @_;
4212
4213         my $volid = $drive->{file};
4214         return if !$volid;
4215
4216         $volhash->{$volid}->{cdrom} //= 1;
4217         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4218
4219         my $replicate = $drive->{replicate} // 1;
4220         $volhash->{$volid}->{replicate} //= 0;
4221         $volhash->{$volid}->{replicate} = 1 if $replicate;
4222
4223         $volhash->{$volid}->{shared} //= 0;
4224         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4225
4226         $volhash->{$volid}->{referenced_in_config} //= 0;
4227         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4228
4229         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4230             if defined($snapname);
4231
4232         my $size = $drive->{size};
4233         $volhash->{$volid}->{size} //= $size if $size;
4234
4235         $volhash->{$volid}->{is_vmstate} //= 0;
4236         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4237
4238         $volhash->{$volid}->{is_unused} //= 0;
4239         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4240     };
4241
4242     my $include_opts = {
4243         extra_keys => ['vmstate'],
4244         include_unused => 1,
4245     };
4246
4247     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4248     foreach my $snapname (keys %{$conf->{snapshots}}) {
4249         my $snap = $conf->{snapshots}->{$snapname};
4250         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4251     }
4252
4253     foreach my $volid (keys %$volhash) {
4254         &$func($volid, $volhash->{$volid}, @param);
4255     }
4256 }
4257
4258 my $fast_plug_option = {
4259     'lock' => 1,
4260     'name' => 1,
4261     'onboot' => 1,
4262     'shares' => 1,
4263     'startup' => 1,
4264     'description' => 1,
4265     'protection' => 1,
4266     'vmstatestorage' => 1,
4267     'hookscript' => 1,
4268     'tags' => 1,
4269 };
4270
4271 # hotplug changes in [PENDING]
4272 # $selection hash can be used to only apply specified options, for
4273 # example: { cores => 1 } (only apply changed 'cores')
4274 # $errors ref is used to return error messages
4275 sub vmconfig_hotplug_pending {
4276     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4277
4278     my $defaults = load_defaults();
4279     my $arch = get_vm_arch($conf);
4280     my $machine_type = get_vm_machine($conf, undef, $arch);
4281
4282     # commit values which do not have any impact on running VM first
4283     # Note: those option cannot raise errors, we we do not care about
4284     # $selection and always apply them.
4285
4286     my $add_error = sub {
4287         my ($opt, $msg) = @_;
4288         $errors->{$opt} = "hotplug problem - $msg";
4289     };
4290
4291     my $changes = 0;
4292     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4293         if ($fast_plug_option->{$opt}) {
4294             $conf->{$opt} = $conf->{pending}->{$opt};
4295             delete $conf->{pending}->{$opt};
4296             $changes = 1;
4297         }
4298     }
4299
4300     if ($changes) {
4301         PVE::QemuConfig->write_config($vmid, $conf);
4302     }
4303
4304     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4305
4306     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4307     foreach my $opt (sort keys %$pending_delete_hash) {
4308         next if $selection && !$selection->{$opt};
4309         my $force = $pending_delete_hash->{$opt}->{force};
4310         eval {
4311             if ($opt eq 'hotplug') {
4312                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4313             } elsif ($opt eq 'tablet') {
4314                 die "skip\n" if !$hotplug_features->{usb};
4315                 if ($defaults->{tablet}) {
4316                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4317                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4318                         if $arch eq 'aarch64';
4319                 } else {
4320                     vm_deviceunplug($vmid, $conf, 'tablet');
4321                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4322                 }
4323             } elsif ($opt =~ m/^usb\d+/) {
4324                 die "skip\n";
4325                 # since we cannot reliably hot unplug usb devices
4326                 # we are disabling it
4327                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4328                 vm_deviceunplug($vmid, $conf, $opt);
4329             } elsif ($opt eq 'vcpus') {
4330                 die "skip\n" if !$hotplug_features->{cpu};
4331                 qemu_cpu_hotplug($vmid, $conf, undef);
4332             } elsif ($opt eq 'balloon') {
4333                 # enable balloon device is not hotpluggable
4334                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4335                 # here we reset the ballooning value to memory
4336                 my $balloon = $conf->{memory} || $defaults->{memory};
4337                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4338             } elsif ($fast_plug_option->{$opt}) {
4339                 # do nothing
4340             } elsif ($opt =~ m/^net(\d+)$/) {
4341                 die "skip\n" if !$hotplug_features->{network};
4342                 vm_deviceunplug($vmid, $conf, $opt);
4343             } elsif (is_valid_drivename($opt)) {
4344                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4345                 vm_deviceunplug($vmid, $conf, $opt);
4346                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4347             } elsif ($opt =~ m/^memory$/) {
4348                 die "skip\n" if !$hotplug_features->{memory};
4349                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4350             } elsif ($opt eq 'cpuunits') {
4351                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4352             } elsif ($opt eq 'cpulimit') {
4353                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4354             } else {
4355                 die "skip\n";
4356             }
4357         };
4358         if (my $err = $@) {
4359             &$add_error($opt, $err) if $err ne "skip\n";
4360         } else {
4361             delete $conf->{$opt};
4362             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4363         }
4364     }
4365
4366     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4367     $apply_pending_cloudinit = sub {
4368         return if $apply_pending_cloudinit_done; # once is enough
4369         $apply_pending_cloudinit_done = 1; # once is enough
4370
4371         my ($key, $value) = @_;
4372
4373         my @cloudinit_opts = keys %$confdesc_cloudinit;
4374         foreach my $opt (keys %{$conf->{pending}}) {
4375             next if !grep { $_ eq $opt } @cloudinit_opts;
4376             $conf->{$opt} = delete $conf->{pending}->{$opt};
4377         }
4378
4379         my $new_conf = { %$conf };
4380         $new_conf->{$key} = $value;
4381         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4382     };
4383
4384     foreach my $opt (keys %{$conf->{pending}}) {
4385         next if $selection && !$selection->{$opt};
4386         my $value = $conf->{pending}->{$opt};
4387         eval {
4388             if ($opt eq 'hotplug') {
4389                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4390             } elsif ($opt eq 'tablet') {
4391                 die "skip\n" if !$hotplug_features->{usb};
4392                 if ($value == 1) {
4393                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4394                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4395                         if $arch eq 'aarch64';
4396                 } elsif ($value == 0) {
4397                     vm_deviceunplug($vmid, $conf, 'tablet');
4398                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4399                 }
4400             } elsif ($opt =~ m/^usb\d+$/) {
4401                 die "skip\n";
4402                 # since we cannot reliably hot unplug usb devices
4403                 # we are disabling it
4404                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4405                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4406                 die "skip\n" if !$d;
4407                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4408             } elsif ($opt eq 'vcpus') {
4409                 die "skip\n" if !$hotplug_features->{cpu};
4410                 qemu_cpu_hotplug($vmid, $conf, $value);
4411             } elsif ($opt eq 'balloon') {
4412                 # enable/disable balloning device is not hotpluggable
4413                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4414                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4415                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4416
4417                 # allow manual ballooning if shares is set to zero
4418                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4419                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4420                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4421                 }
4422             } elsif ($opt =~ m/^net(\d+)$/) {
4423                 # some changes can be done without hotplug
4424                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4425                                     $vmid, $opt, $value, $arch, $machine_type);
4426             } elsif (is_valid_drivename($opt)) {
4427                 die "skip\n" if $opt eq 'efidisk0';
4428                 # some changes can be done without hotplug
4429                 my $drive = parse_drive($opt, $value);
4430                 if (drive_is_cloudinit($drive)) {
4431                     &$apply_pending_cloudinit($opt, $value);
4432                 }
4433                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4434                                      $vmid, $opt, $value, $arch, $machine_type);
4435             } elsif ($opt =~ m/^memory$/) { #dimms
4436                 die "skip\n" if !$hotplug_features->{memory};
4437                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4438             } elsif ($opt eq 'cpuunits') {
4439                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4440             } elsif ($opt eq 'cpulimit') {
4441                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4442                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4443             } else {
4444                 die "skip\n";  # skip non-hot-pluggable options
4445             }
4446         };
4447         if (my $err = $@) {
4448             &$add_error($opt, $err) if $err ne "skip\n";
4449         } else {
4450             $conf->{$opt} = $value;
4451             delete $conf->{pending}->{$opt};
4452         }
4453     }
4454
4455     PVE::QemuConfig->write_config($vmid, $conf);
4456 }
4457
4458 sub try_deallocate_drive {
4459     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4460
4461     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4462         my $volid = $drive->{file};
4463         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4464             my $sid = PVE::Storage::parse_volume_id($volid);
4465             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4466
4467             # check if the disk is really unused
4468             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4469                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4470             PVE::Storage::vdisk_free($storecfg, $volid);
4471             return 1;
4472         } else {
4473             # If vm is not owner of this disk remove from config
4474             return 1;
4475         }
4476     }
4477
4478     return undef;
4479 }
4480
4481 sub vmconfig_delete_or_detach_drive {
4482     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4483
4484     my $drive = parse_drive($opt, $conf->{$opt});
4485
4486     my $rpcenv = PVE::RPCEnvironment::get();
4487     my $authuser = $rpcenv->get_user();
4488
4489     if ($force) {
4490         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4491         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4492     } else {
4493         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4494     }
4495 }
4496
4497
4498
4499 sub vmconfig_apply_pending {
4500     my ($vmid, $conf, $storecfg, $errors) = @_;
4501
4502     my $add_apply_error = sub {
4503         my ($opt, $msg) = @_;
4504         my $err_msg = "unable to apply pending change $opt : $msg";
4505         $errors->{$opt} = $err_msg;
4506         warn $err_msg;
4507     };
4508
4509     # cold plug
4510
4511     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4512     foreach my $opt (sort keys %$pending_delete_hash) {
4513         my $force = $pending_delete_hash->{$opt}->{force};
4514         eval {
4515             if ($opt =~ m/^unused/) {
4516                 die "internal error";
4517             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4518                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4519             }
4520         };
4521         if (my $err = $@) {
4522             $add_apply_error->($opt, $err);
4523         } else {
4524             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4525             delete $conf->{$opt};
4526         }
4527     }
4528
4529     PVE::QemuConfig->cleanup_pending($conf);
4530
4531     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4532         next if $opt eq 'delete'; # just to be sure
4533         eval {
4534             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4535                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4536             }
4537         };
4538         if (my $err = $@) {
4539             $add_apply_error->($opt, $err);
4540         } else {
4541             $conf->{$opt} = delete $conf->{pending}->{$opt};
4542         }
4543     }
4544
4545     # write all changes at once to avoid unnecessary i/o
4546     PVE::QemuConfig->write_config($vmid, $conf);
4547 }
4548
4549 sub vmconfig_update_net {
4550     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4551
4552     my $newnet = parse_net($value);
4553
4554     if ($conf->{$opt}) {
4555         my $oldnet = parse_net($conf->{$opt});
4556
4557         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4558             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4559             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4560             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4561
4562             # for non online change, we try to hot-unplug
4563             die "skip\n" if !$hotplug;
4564             vm_deviceunplug($vmid, $conf, $opt);
4565         } else {
4566
4567             die "internal error" if $opt !~ m/net(\d+)/;
4568             my $iface = "tap${vmid}i$1";
4569
4570             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4571                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4572                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4573                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4574                 PVE::Network::tap_unplug($iface);
4575
4576                 if ($have_sdn) {
4577                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4578                 } else {
4579                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4580                 }
4581             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4582                 # Rate can be applied on its own but any change above needs to
4583                 # include the rate in tap_plug since OVS resets everything.
4584                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4585             }
4586
4587             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4588                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4589             }
4590
4591             return 1;
4592         }
4593     }
4594
4595     if ($hotplug) {
4596         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4597     } else {
4598         die "skip\n";
4599     }
4600 }
4601
4602 sub vmconfig_update_disk {
4603     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4604
4605     my $drive = parse_drive($opt, $value);
4606
4607     if ($conf->{$opt}) {
4608
4609         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4610
4611             my $media = $drive->{media} || 'disk';
4612             my $oldmedia = $old_drive->{media} || 'disk';
4613             die "unable to change media type\n" if $media ne $oldmedia;
4614
4615             if (!drive_is_cdrom($old_drive)) {
4616
4617                 if ($drive->{file} ne $old_drive->{file}) {
4618
4619                     die "skip\n" if !$hotplug;
4620
4621                     # unplug and register as unused
4622                     vm_deviceunplug($vmid, $conf, $opt);
4623                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4624
4625                 } else {
4626                     # update existing disk
4627
4628                     # skip non hotpluggable value
4629                     if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4630                         safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4631                         safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4632                         safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
4633                         safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
4634                         die "skip\n";
4635                     }
4636
4637                     # apply throttle
4638                     if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4639                         safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4640                         safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4641                         safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4642                         safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4643                         safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4644                         safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4645                         safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4646                         safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4647                         safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4648                         safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4649                         safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4650                         safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4651                         safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4652                         safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4653                         safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4654                         safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4655                         safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4656
4657                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4658                                                    ($drive->{mbps} || 0)*1024*1024,
4659                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4660                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4661                                                    $drive->{iops} || 0,
4662                                                    $drive->{iops_rd} || 0,
4663                                                    $drive->{iops_wr} || 0,
4664                                                    ($drive->{mbps_max} || 0)*1024*1024,
4665                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4666                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4667                                                    $drive->{iops_max} || 0,
4668                                                    $drive->{iops_rd_max} || 0,
4669                                                    $drive->{iops_wr_max} || 0,
4670                                                    $drive->{bps_max_length} || 1,
4671                                                    $drive->{bps_rd_max_length} || 1,
4672                                                    $drive->{bps_wr_max_length} || 1,
4673                                                    $drive->{iops_max_length} || 1,
4674                                                    $drive->{iops_rd_max_length} || 1,
4675                                                    $drive->{iops_wr_max_length} || 1);
4676
4677                     }
4678
4679                     return 1;
4680                 }
4681
4682             } else { # cdrom
4683
4684                 if ($drive->{file} eq 'none') {
4685                     mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4686                     if (drive_is_cloudinit($old_drive)) {
4687                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4688                     }
4689                 } else {
4690                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4691
4692                     # force eject if locked
4693                     mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4694
4695                     if ($path) {
4696                         mon_cmd($vmid, "blockdev-change-medium",
4697                             id => "$opt", filename => "$path");
4698                     }
4699                 }
4700
4701                 return 1;
4702             }
4703         }
4704     }
4705
4706     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4707     # hotplug new disks
4708     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4709     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
4710 }
4711
4712 # called in locked context by incoming migration
4713 sub vm_migrate_get_nbd_disks {
4714     my ($storecfg, $conf, $replicated_volumes) = @_;
4715
4716     my $local_volumes = {};
4717     PVE::QemuConfig->foreach_volume($conf, sub {
4718         my ($ds, $drive) = @_;
4719
4720         return if drive_is_cdrom($drive);
4721
4722         my $volid = $drive->{file};
4723
4724         return if !$volid;
4725
4726         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4727
4728         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4729         return if $scfg->{shared};
4730
4731         # replicated disks re-use existing state via bitmap
4732         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
4733         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
4734     });
4735     return $local_volumes;
4736 }
4737
4738 # called in locked context by incoming migration
4739 sub vm_migrate_alloc_nbd_disks {
4740     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
4741
4742     my $format = undef;
4743
4744     my $nbd = {};
4745     foreach my $opt (sort keys %$source_volumes) {
4746         my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
4747
4748         if ($use_existing) {
4749             $nbd->{$opt}->{drivestr} = print_drive($drive);
4750             $nbd->{$opt}->{volid} = $volid;
4751             $nbd->{$opt}->{replicated} = 1;
4752             next;
4753         }
4754
4755         # If a remote storage is specified and the format of the original
4756         # volume is not available there, fall back to the default format.
4757         # Otherwise use the same format as the original.
4758         if (!$storagemap->{identity}) {
4759             $storeid = map_storage($storagemap, $storeid);
4760             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4761             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4762             my $fileFormat = qemu_img_format($scfg, $volname);
4763             $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
4764         } else {
4765             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4766             $format = qemu_img_format($scfg, $volname);
4767         }
4768
4769         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4770         my $newdrive = $drive;
4771         $newdrive->{format} = $format;
4772         $newdrive->{file} = $newvolid;
4773         my $drivestr = print_drive($newdrive);
4774         $nbd->{$opt}->{drivestr} = $drivestr;
4775         $nbd->{$opt}->{volid} = $newvolid;
4776     }
4777
4778     return $nbd;
4779 }
4780
4781 # see vm_start_nolock for parameters, additionally:
4782 # migrate_opts:
4783 #   storagemap = parsed storage map for allocating NBD disks
4784 sub vm_start {
4785     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
4786
4787     return PVE::QemuConfig->lock_config($vmid, sub {
4788         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
4789
4790         die "you can't start a vm if it's a template\n"
4791             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
4792
4793         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
4794
4795         PVE::QemuConfig->check_lock($conf)
4796             if !($params->{skiplock} || $has_suspended_lock);
4797
4798         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
4799
4800         die "VM $vmid already running\n" if check_running($vmid, undef, $migrate_opts->{migratedfrom});
4801
4802         if (my $storagemap = $migrate_opts->{storagemap}) {
4803             my $replicated = $migrate_opts->{replicated_volumes};
4804             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
4805             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
4806
4807             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
4808                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
4809             }
4810         }
4811
4812         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
4813     });
4814 }
4815
4816
4817 # params:
4818 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
4819 #   skiplock => 0/1, skip checking for config lock
4820 #   skiptemplate => 0/1, skip checking whether VM is template
4821 #   forcemachine => to force Qemu machine (rollback/migration)
4822 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
4823 #   timeout => in seconds
4824 #   paused => start VM in paused state (backup)
4825 #   resume => resume from hibernation
4826 # migrate_opts:
4827 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
4828 #   migratedfrom => source node
4829 #   spice_ticket => used for spice migration, passed via tunnel/stdin
4830 #   network => CIDR of migration network
4831 #   type => secure/insecure - tunnel over encrypted connection or plain-text
4832 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
4833 #   replicated_volumes = which volids should be re-used with bitmaps for nbd migration
4834 sub vm_start_nolock {
4835     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
4836
4837     my $statefile = $params->{statefile};
4838     my $resume = $params->{resume};
4839
4840     my $migratedfrom = $migrate_opts->{migratedfrom};
4841     my $migration_type = $migrate_opts->{type};
4842
4843     my $res = {};
4844
4845     # clean up leftover reboot request files
4846     eval { clear_reboot_request($vmid); };
4847     warn $@ if $@;
4848
4849     if (!$statefile && scalar(keys %{$conf->{pending}})) {
4850         vmconfig_apply_pending($vmid, $conf, $storecfg);
4851         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4852     }
4853
4854     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4855
4856     my $defaults = load_defaults();
4857
4858     # set environment variable useful inside network script
4859     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4860
4861     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
4862
4863     my $forcemachine = $params->{forcemachine};
4864     my $forcecpu = $params->{forcecpu};
4865     if ($resume) {
4866         # enforce machine and CPU type on suspended vm to ensure HW compatibility
4867         $forcemachine = $conf->{runningmachine};
4868         $forcecpu = $conf->{runningcpu};
4869         print "Resuming suspended VM\n";
4870     }
4871
4872     my ($cmd, $vollist, $spice_port) =
4873         config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
4874
4875     my $migration_ip;
4876     my $get_migration_ip = sub {
4877         my ($nodename) = @_;
4878
4879         return $migration_ip if defined($migration_ip);
4880
4881         my $cidr = $migrate_opts->{network};
4882
4883         if (!defined($cidr)) {
4884             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4885             $cidr = $dc_conf->{migration}->{network};
4886         }
4887
4888         if (defined($cidr)) {
4889             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
4890
4891             die "could not get IP: no address configured on local " .
4892                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
4893
4894             die "could not get IP: multiple addresses configured on local " .
4895                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
4896
4897             $migration_ip = @$ips[0];
4898         }
4899
4900         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
4901             if !defined($migration_ip);
4902
4903         return $migration_ip;
4904     };
4905
4906     my $migrate_uri;
4907     if ($statefile) {
4908         if ($statefile eq 'tcp') {
4909             my $localip = "localhost";
4910             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4911             my $nodename = nodename();
4912
4913             if (!defined($migration_type)) {
4914                 if (defined($datacenterconf->{migration}->{type})) {
4915                     $migration_type = $datacenterconf->{migration}->{type};
4916                 } else {
4917                     $migration_type = 'secure';
4918                 }
4919             }
4920
4921             if ($migration_type eq 'insecure') {
4922                 $localip = $get_migration_ip->($nodename);
4923                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4924             }
4925
4926             my $pfamily = PVE::Tools::get_host_address_family($nodename);
4927             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4928             $migrate_uri = "tcp:${localip}:${migrate_port}";
4929             push @$cmd, '-incoming', $migrate_uri;
4930             push @$cmd, '-S';
4931
4932         } elsif ($statefile eq 'unix') {
4933             # should be default for secure migrations as a ssh TCP forward
4934             # tunnel is not deterministic reliable ready and fails regurarly
4935             # to set up in time, so use UNIX socket forwards
4936             my $socket_addr = "/run/qemu-server/$vmid.migrate";
4937             unlink $socket_addr;
4938
4939             $migrate_uri = "unix:$socket_addr";
4940
4941             push @$cmd, '-incoming', $migrate_uri;
4942             push @$cmd, '-S';
4943
4944         } elsif (-e $statefile) {
4945             push @$cmd, '-loadstate', $statefile;
4946         } else {
4947             my $statepath = PVE::Storage::path($storecfg, $statefile);
4948             push @$vollist, $statefile;
4949             push @$cmd, '-loadstate', $statepath;
4950         }
4951     } elsif ($params->{paused}) {
4952         push @$cmd, '-S';
4953     }
4954
4955     # host pci devices
4956     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
4957       my $d = parse_hostpci($conf->{"hostpci$i"});
4958       next if !$d;
4959       my $pcidevices = $d->{pciid};
4960       foreach my $pcidevice (@$pcidevices) {
4961             my $pciid = $pcidevice->{id};
4962
4963             my $info = PVE::SysFSTools::pci_device_info("$pciid");
4964             die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
4965             die "no pci device info for device '$pciid'\n" if !$info;
4966
4967             if ($d->{mdev}) {
4968                 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
4969                 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
4970             } else {
4971                 die "can't unbind/bind pci group to vfio '$pciid'\n"
4972                     if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
4973                 die "can't reset pci device '$pciid'\n"
4974                     if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
4975             }
4976       }
4977     }
4978
4979     PVE::Storage::activate_volumes($storecfg, $vollist);
4980
4981     eval {
4982         run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
4983             outfunc => sub {}, errfunc => sub {});
4984     };
4985     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
4986     # timeout should be more than enough here...
4987     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
4988
4989     my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4990                                               : $defaults->{cpuunits};
4991
4992     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
4993     my %run_params = (
4994         timeout => $statefile ? undef : $start_timeout,
4995         umask => 0077,
4996         noerr => 1,
4997     );
4998
4999     # when migrating, prefix QEMU output so other side can pick up any
5000     # errors that might occur and show the user
5001     if ($migratedfrom) {
5002         $run_params{quiet} = 1;
5003         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5004     }
5005
5006     my %properties = (
5007         Slice => 'qemu.slice',
5008         KillMode => 'none',
5009         CPUShares => $cpuunits
5010     );
5011
5012     if (my $cpulimit = $conf->{cpulimit}) {
5013         $properties{CPUQuota} = int($cpulimit * 100);
5014     }
5015     $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5016
5017     my $run_qemu = sub {
5018         PVE::Tools::run_fork sub {
5019             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5020
5021             my $exitcode = run_command($cmd, %run_params);
5022             die "QEMU exited with code $exitcode\n" if $exitcode;
5023         };
5024     };
5025
5026     if ($conf->{hugepages}) {
5027
5028         my $code = sub {
5029             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5030             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5031
5032             PVE::QemuServer::Memory::hugepages_mount();
5033             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5034
5035             eval { $run_qemu->() };
5036             if (my $err = $@) {
5037                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5038                 die $err;
5039             }
5040
5041             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5042         };
5043         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5044
5045     } else {
5046         eval { $run_qemu->() };
5047     }
5048
5049     if (my $err = $@) {
5050         # deactivate volumes if start fails
5051         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5052         die "start failed: $err";
5053     }
5054
5055     print "migration listens on $migrate_uri\n" if $migrate_uri;
5056     $res->{migrate_uri} = $migrate_uri;
5057
5058     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5059         eval { mon_cmd($vmid, "cont"); };
5060         warn $@ if $@;
5061     }
5062
5063     #start nbd server for storage migration
5064     if (my $nbd = $migrate_opts->{nbd}) {
5065         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5066
5067         my $migrate_storage_uri;
5068         # nbd_protocol_version > 0 for unix socket support
5069         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5070             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5071             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5072             $migrate_storage_uri = "nbd:unix:$socket_path";
5073         } else {
5074             my $nodename = nodename();
5075             my $localip = $get_migration_ip->($nodename);
5076             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5077             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5078
5079             mon_cmd($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${storage_migrate_port}" } } );
5080             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5081             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5082         }
5083
5084         $res->{migrate_storage_uri} = $migrate_storage_uri;
5085
5086         foreach my $opt (sort keys %$nbd) {
5087             my $drivestr = $nbd->{$opt}->{drivestr};
5088             my $volid = $nbd->{$opt}->{volid};
5089             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5090             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5091             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5092             print "re-using replicated volume: $opt - $volid\n"
5093                 if $nbd->{$opt}->{replicated};
5094
5095             $res->{drives}->{$opt} = $nbd->{$opt};
5096             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5097         }
5098     }
5099
5100     if ($migratedfrom) {
5101         eval {
5102             set_migration_caps($vmid);
5103         };
5104         warn $@ if $@;
5105
5106         if ($spice_port) {
5107             print "spice listens on port $spice_port\n";
5108             $res->{spice_port} = $spice_port;
5109             if ($migrate_opts->{spice_ticket}) {
5110                 mon_cmd($vmid, "set_password", protocol => 'spice', password => $migrate_opts->{spice_ticket});
5111                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5112             }
5113         }
5114
5115     } else {
5116         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5117             if !$statefile && $conf->{balloon};
5118
5119         foreach my $opt (keys %$conf) {
5120             next if $opt !~  m/^net\d+$/;
5121             my $nicconf = parse_net($conf->{$opt});
5122             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5123         }
5124     }
5125
5126     mon_cmd($vmid, 'qom-set',
5127                 path => "machine/peripheral/balloon0",
5128                 property => "guest-stats-polling-interval",
5129                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5130
5131     if ($resume) {
5132         print "Resumed VM, removing state\n";
5133         if (my $vmstate = $conf->{vmstate}) {
5134             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5135             PVE::Storage::vdisk_free($storecfg, $vmstate);
5136         }
5137         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5138         PVE::QemuConfig->write_config($vmid, $conf);
5139     }
5140
5141     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5142
5143     return $res;
5144 }
5145
5146 sub vm_commandline {
5147     my ($storecfg, $vmid, $snapname) = @_;
5148
5149     my $conf = PVE::QemuConfig->load_config($vmid);
5150     my $forcemachine;
5151     my $forcecpu;
5152
5153     if ($snapname) {
5154         my $snapshot = $conf->{snapshots}->{$snapname};
5155         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5156
5157         # check for machine or CPU overrides in snapshot
5158         $forcemachine = $snapshot->{runningmachine};
5159         $forcecpu = $snapshot->{runningcpu};
5160
5161         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5162
5163         $conf = $snapshot;
5164     }
5165
5166     my $defaults = load_defaults();
5167
5168     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults,
5169         $forcemachine, $forcecpu);
5170
5171     return PVE::Tools::cmd2string($cmd);
5172 }
5173
5174 sub vm_reset {
5175     my ($vmid, $skiplock) = @_;
5176
5177     PVE::QemuConfig->lock_config($vmid, sub {
5178
5179         my $conf = PVE::QemuConfig->load_config($vmid);
5180
5181         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5182
5183         mon_cmd($vmid, "system_reset");
5184     });
5185 }
5186
5187 sub get_vm_volumes {
5188     my ($conf) = @_;
5189
5190     my $vollist = [];
5191     foreach_volid($conf, sub {
5192         my ($volid, $attr) = @_;
5193
5194         return if $volid =~ m|^/|;
5195
5196         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5197         return if !$sid;
5198
5199         push @$vollist, $volid;
5200     });
5201
5202     return $vollist;
5203 }
5204
5205 sub vm_stop_cleanup {
5206     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5207
5208     eval {
5209
5210         if (!$keepActive) {
5211             my $vollist = get_vm_volumes($conf);
5212             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5213         }
5214
5215         foreach my $ext (qw(mon qmp pid vnc qga)) {
5216             unlink "/var/run/qemu-server/${vmid}.$ext";
5217         }
5218
5219         if ($conf->{ivshmem}) {
5220             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5221             # just delete it for now, VMs which have this already open do not
5222             # are affected, but new VMs will get a separated one. If this
5223             # becomes an issue we either add some sort of ref-counting or just
5224             # add a "don't delete on stop" flag to the ivshmem format.
5225             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5226         }
5227
5228         foreach my $key (keys %$conf) {
5229             next if $key !~ m/^hostpci(\d+)$/;
5230             my $hostpciindex = $1;
5231             my $d = parse_hostpci($conf->{$key});
5232             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5233
5234             foreach my $pci (@{$d->{pciid}}) {
5235                 my $pciid = $pci->{id};
5236                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5237             }
5238         }
5239
5240         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5241     };
5242     warn $@ if $@; # avoid errors - just warn
5243 }
5244
5245 # call only in locked context
5246 sub _do_vm_stop {
5247     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5248
5249     my $pid = check_running($vmid, $nocheck);
5250     return if !$pid;
5251
5252     my $conf;
5253     if (!$nocheck) {
5254         $conf = PVE::QemuConfig->load_config($vmid);
5255         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5256         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5257             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5258             $timeout = $opts->{down} if $opts->{down};
5259         }
5260         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5261     }
5262
5263     eval {
5264         if ($shutdown) {
5265             if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5266                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5267             } else {
5268                 mon_cmd($vmid, "system_powerdown");
5269             }
5270         } else {
5271             mon_cmd($vmid, "quit");
5272         }
5273     };
5274     my $err = $@;
5275
5276     if (!$err) {
5277         $timeout = 60 if !defined($timeout);
5278
5279         my $count = 0;
5280         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5281             $count++;
5282             sleep 1;
5283         }
5284
5285         if ($count >= $timeout) {
5286             if ($force) {
5287                 warn "VM still running - terminating now with SIGTERM\n";
5288                 kill 15, $pid;
5289             } else {
5290                 die "VM quit/powerdown failed - got timeout\n";
5291             }
5292         } else {
5293             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5294             return;
5295         }
5296     } else {
5297        if (!check_running($vmid, $nocheck)) {
5298             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5299             return;
5300        }
5301        if ($force) {
5302             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5303             kill 15, $pid;
5304         } else {
5305             die "VM quit/powerdown failed\n";
5306         }
5307     }
5308
5309     # wait again
5310     $timeout = 10;
5311
5312     my $count = 0;
5313     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5314         $count++;
5315         sleep 1;
5316     }
5317
5318     if ($count >= $timeout) {
5319         warn "VM still running - terminating now with SIGKILL\n";
5320         kill 9, $pid;
5321         sleep 1;
5322     }
5323
5324     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5325 }
5326
5327 # Note: use $nocheck to skip tests if VM configuration file exists.
5328 # We need that when migration VMs to other nodes (files already moved)
5329 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5330 sub vm_stop {
5331     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5332
5333     $force = 1 if !defined($force) && !$shutdown;
5334
5335     if ($migratedfrom){
5336         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5337         kill 15, $pid if $pid;
5338         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5339         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5340         return;
5341     }
5342
5343     PVE::QemuConfig->lock_config($vmid, sub {
5344         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5345    });
5346 }
5347
5348 sub vm_reboot {
5349     my ($vmid, $timeout) = @_;
5350
5351     PVE::QemuConfig->lock_config($vmid, sub {
5352         eval {
5353
5354             # only reboot if running, as qmeventd starts it again on a stop event
5355             return if !check_running($vmid);
5356
5357             create_reboot_request($vmid);
5358
5359             my $storecfg = PVE::Storage::config();
5360             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5361
5362         };
5363         if (my $err = $@) {
5364             # avoid that the next normal shutdown will be confused for a reboot
5365             clear_reboot_request($vmid);
5366             die $err;
5367         }
5368    });
5369 }
5370
5371 # note: if using the statestorage parameter, the caller has to check privileges
5372 sub vm_suspend {
5373     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5374
5375     my $conf;
5376     my $path;
5377     my $storecfg;
5378     my $vmstate;
5379
5380     PVE::QemuConfig->lock_config($vmid, sub {
5381
5382         $conf = PVE::QemuConfig->load_config($vmid);
5383
5384         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5385         PVE::QemuConfig->check_lock($conf)
5386             if !($skiplock || $is_backing_up);
5387
5388         die "cannot suspend to disk during backup\n"
5389             if $is_backing_up && $includestate;
5390
5391         if ($includestate) {
5392             $conf->{lock} = 'suspending';
5393             my $date = strftime("%Y-%m-%d", localtime(time()));
5394             $storecfg = PVE::Storage::config();
5395             if (!$statestorage) {
5396                 $statestorage = find_vmstate_storage($conf, $storecfg);
5397                 # check permissions for the storage
5398                 my $rpcenv = PVE::RPCEnvironment::get();
5399                 if ($rpcenv->{type} ne 'cli') {
5400                     my $authuser = $rpcenv->get_user();
5401                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5402                 }
5403             }
5404
5405
5406             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5407             $path = PVE::Storage::path($storecfg, $vmstate);
5408             PVE::QemuConfig->write_config($vmid, $conf);
5409         } else {
5410             mon_cmd($vmid, "stop");
5411         }
5412     });
5413
5414     if ($includestate) {
5415         # save vm state
5416         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5417
5418         eval {
5419             mon_cmd($vmid, "savevm-start", statefile => $path);
5420             for(;;) {
5421                 my $state = mon_cmd($vmid, "query-savevm");
5422                 if (!$state->{status}) {
5423                     die "savevm not active\n";
5424                 } elsif ($state->{status} eq 'active') {
5425                     sleep(1);
5426                     next;
5427                 } elsif ($state->{status} eq 'completed') {
5428                     print "State saved, quitting\n";
5429                     last;
5430                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5431                     die "query-savevm failed with error '$state->{error}'\n"
5432                 } else {
5433                     die "query-savevm returned status '$state->{status}'\n";
5434                 }
5435             }
5436         };
5437         my $err = $@;
5438
5439         PVE::QemuConfig->lock_config($vmid, sub {
5440             $conf = PVE::QemuConfig->load_config($vmid);
5441             if ($err) {
5442                 # cleanup, but leave suspending lock, to indicate something went wrong
5443                 eval {
5444                     mon_cmd($vmid, "savevm-end");
5445                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5446                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5447                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
5448                     PVE::QemuConfig->write_config($vmid, $conf);
5449                 };
5450                 warn $@ if $@;
5451                 die $err;
5452             }
5453
5454             die "lock changed unexpectedly\n"
5455                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5456
5457             mon_cmd($vmid, "quit");
5458             $conf->{lock} = 'suspended';
5459             PVE::QemuConfig->write_config($vmid, $conf);
5460         });
5461     }
5462 }
5463
5464 sub vm_resume {
5465     my ($vmid, $skiplock, $nocheck) = @_;
5466
5467     PVE::QemuConfig->lock_config($vmid, sub {
5468         my $res = mon_cmd($vmid, 'query-status');
5469         my $resume_cmd = 'cont';
5470
5471         if ($res->{status} && $res->{status} eq 'suspended') {
5472             $resume_cmd = 'system_wakeup';
5473         }
5474
5475         if (!$nocheck) {
5476
5477             my $conf = PVE::QemuConfig->load_config($vmid);
5478
5479             PVE::QemuConfig->check_lock($conf)
5480                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5481         }
5482
5483         mon_cmd($vmid, $resume_cmd);
5484     });
5485 }
5486
5487 sub vm_sendkey {
5488     my ($vmid, $skiplock, $key) = @_;
5489
5490     PVE::QemuConfig->lock_config($vmid, sub {
5491
5492         my $conf = PVE::QemuConfig->load_config($vmid);
5493
5494         # there is no qmp command, so we use the human monitor command
5495         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5496         die $res if $res ne '';
5497     });
5498 }
5499
5500 # vzdump restore implementaion
5501
5502 sub tar_archive_read_firstfile {
5503     my $archive = shift;
5504
5505     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5506
5507     # try to detect archive type first
5508     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5509         die "unable to open file '$archive'\n";
5510     my $firstfile = <$fh>;
5511     kill 15, $pid;
5512     close $fh;
5513
5514     die "ERROR: archive contaions no data\n" if !$firstfile;
5515     chomp $firstfile;
5516
5517     return $firstfile;
5518 }
5519
5520 sub tar_restore_cleanup {
5521     my ($storecfg, $statfile) = @_;
5522
5523     print STDERR "starting cleanup\n";
5524
5525     if (my $fd = IO::File->new($statfile, "r")) {
5526         while (defined(my $line = <$fd>)) {
5527             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5528                 my $volid = $2;
5529                 eval {
5530                     if ($volid =~ m|^/|) {
5531                         unlink $volid || die 'unlink failed\n';
5532                     } else {
5533                         PVE::Storage::vdisk_free($storecfg, $volid);
5534                     }
5535                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5536                 };
5537                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5538             } else {
5539                 print STDERR "unable to parse line in statfile - $line";
5540             }
5541         }
5542         $fd->close();
5543     }
5544 }
5545
5546 sub restore_file_archive {
5547     my ($archive, $vmid, $user, $opts) = @_;
5548
5549     return restore_vma_archive($archive, $vmid, $user, $opts)
5550         if $archive eq '-';
5551
5552     my $info = PVE::Storage::archive_info($archive);
5553     my $format = $opts->{format} // $info->{format};
5554     my $comp = $info->{compression};
5555
5556     # try to detect archive format
5557     if ($format eq 'tar') {
5558         return restore_tar_archive($archive, $vmid, $user, $opts);
5559     } else {
5560         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5561     }
5562 }
5563
5564 # hepler to remove disks that will not be used after restore
5565 my $restore_cleanup_oldconf = sub {
5566     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
5567
5568     PVE::QemuConfig->foreach_volume($oldconf, sub {
5569         my ($ds, $drive) = @_;
5570
5571         return if drive_is_cdrom($drive, 1);
5572
5573         my $volid = $drive->{file};
5574         return if !$volid || $volid =~ m|^/|;
5575
5576         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
5577         return if !$path || !$owner || ($owner != $vmid);
5578
5579         # Note: only delete disk we want to restore
5580         # other volumes will become unused
5581         if ($virtdev_hash->{$ds}) {
5582             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
5583             if (my $err = $@) {
5584                 warn $err;
5585             }
5586         }
5587     });
5588
5589     # delete vmstate files, after the restore we have no snapshots anymore
5590     foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5591         my $snap = $oldconf->{snapshots}->{$snapname};
5592         if ($snap->{vmstate}) {
5593             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
5594             if (my $err = $@) {
5595                 warn $err;
5596             }
5597         }
5598     }
5599 };
5600
5601 # Helper to parse vzdump backup device hints
5602 #
5603 # $rpcenv: Environment, used to ckeck storage permissions
5604 # $user: User ID, to check storage permissions
5605 # $storecfg: Storage configuration
5606 # $fh: the file handle for reading the configuration
5607 # $devinfo: should contain device sizes for all backu-up'ed devices
5608 # $options: backup options (pool, default storage)
5609 #
5610 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
5611 my $parse_backup_hints = sub {
5612     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
5613
5614     my $virtdev_hash = {};
5615
5616     while (defined(my $line = <$fh>)) {
5617         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5618             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5619             die "archive does not contain data for drive '$virtdev'\n"
5620                 if !$devinfo->{$devname};
5621
5622             if (defined($options->{storage})) {
5623                 $storeid = $options->{storage} || 'local';
5624             } elsif (!$storeid) {
5625                 $storeid = 'local';
5626             }
5627             $format = 'raw' if !$format;
5628             $devinfo->{$devname}->{devname} = $devname;
5629             $devinfo->{$devname}->{virtdev} = $virtdev;
5630             $devinfo->{$devname}->{format} = $format;
5631             $devinfo->{$devname}->{storeid} = $storeid;
5632
5633             # check permission on storage
5634             my $pool = $options->{pool}; # todo: do we need that?
5635             if ($user ne 'root@pam') {
5636                 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5637             }
5638
5639             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5640         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
5641             my $virtdev = $1;
5642             my $drive = parse_drive($virtdev, $2);
5643             if (drive_is_cloudinit($drive)) {
5644                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5645                 $storeid = $options->{storage} if defined ($options->{storage});
5646                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5647                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
5648
5649                 $virtdev_hash->{$virtdev} = {
5650                     format => $format,
5651                     storeid => $storeid,
5652                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
5653                     is_cloudinit => 1,
5654                 };
5655             }
5656         }
5657     }
5658
5659     return $virtdev_hash;
5660 };
5661
5662 # Helper to allocate and activate all volumes required for a restore
5663 #
5664 # $storecfg: Storage configuration
5665 # $virtdev_hash: as returned by parse_backup_hints()
5666 #
5667 # Returns: { $virtdev => $volid }
5668 my $restore_allocate_devices = sub {
5669     my ($storecfg, $virtdev_hash, $vmid) = @_;
5670
5671     my $map = {};
5672     foreach my $virtdev (sort keys %$virtdev_hash) {
5673         my $d = $virtdev_hash->{$virtdev};
5674         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5675         my $storeid = $d->{storeid};
5676         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5677
5678         # test if requested format is supported
5679         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5680         my $supported = grep { $_ eq $d->{format} } @$validFormats;
5681         $d->{format} = $defFormat if !$supported;
5682
5683         my $name;
5684         if ($d->{is_cloudinit}) {
5685             $name = "vm-$vmid-cloudinit";
5686             $name .= ".$d->{format}" if $d->{format} ne 'raw';
5687         }
5688
5689         my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
5690
5691         print STDERR "new volume ID is '$volid'\n";
5692         $d->{volid} = $volid;
5693
5694         PVE::Storage::activate_volumes($storecfg, [$volid]);
5695
5696         $map->{$virtdev} = $volid;
5697     }
5698
5699     return $map;
5700 };
5701
5702 my $restore_update_config_line = sub {
5703     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5704
5705     return if $line =~ m/^\#qmdump\#/;
5706     return if $line =~ m/^\#vzdump\#/;
5707     return if $line =~ m/^lock:/;
5708     return if $line =~ m/^unused\d+:/;
5709     return if $line =~ m/^parent:/;
5710
5711     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5712     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5713         # try to convert old 1.X settings
5714         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5715         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5716             my ($model, $macaddr) = split(/\=/, $devconfig);
5717             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5718             my $net = {
5719                 model => $model,
5720                 bridge => "vmbr$ind",
5721                 macaddr => $macaddr,
5722             };
5723             my $netstr = print_net($net);
5724
5725             print $outfd "net$cookie->{netcount}: $netstr\n";
5726             $cookie->{netcount}++;
5727         }
5728     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5729         my ($id, $netstr) = ($1, $2);
5730         my $net = parse_net($netstr);
5731         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5732         $netstr = print_net($net);
5733         print $outfd "$id: $netstr\n";
5734     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5735         my $virtdev = $1;
5736         my $value = $3;
5737         my $di = parse_drive($virtdev, $value);
5738         if (defined($di->{backup}) && !$di->{backup}) {
5739             print $outfd "#$line";
5740         } elsif ($map->{$virtdev}) {
5741             delete $di->{format}; # format can change on restore
5742             $di->{file} = $map->{$virtdev};
5743             $value = print_drive($di);
5744             print $outfd "$virtdev: $value\n";
5745         } else {
5746             print $outfd $line;
5747         }
5748     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5749         my $vmgenid = $1;
5750         if ($vmgenid ne '0') {
5751             # always generate a new vmgenid if there was a valid one setup
5752             $vmgenid = generate_uuid();
5753         }
5754         print $outfd "vmgenid: $vmgenid\n";
5755     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5756         my ($uuid, $uuid_str);
5757         UUID::generate($uuid);
5758         UUID::unparse($uuid, $uuid_str);
5759         my $smbios1 = parse_smbios1($2);
5760         $smbios1->{uuid} = $uuid_str;
5761         print $outfd $1.print_smbios1($smbios1)."\n";
5762     } else {
5763         print $outfd $line;
5764     }
5765 };
5766
5767 my $restore_deactivate_volumes = sub {
5768     my ($storecfg, $devinfo) = @_;
5769
5770     my $vollist = [];
5771     foreach my $devname (keys %$devinfo) {
5772         my $volid = $devinfo->{$devname}->{volid};
5773         push @$vollist, $volid if $volid;
5774     }
5775
5776     PVE::Storage::deactivate_volumes($storecfg, $vollist);
5777 };
5778
5779 my $restore_destroy_volumes = sub {
5780     my ($storecfg, $devinfo) = @_;
5781
5782     foreach my $devname (keys %$devinfo) {
5783         my $volid = $devinfo->{$devname}->{volid};
5784         next if !$volid;
5785         eval {
5786             if ($volid =~ m|^/|) {
5787                 unlink $volid || die 'unlink failed\n';
5788             } else {
5789                 PVE::Storage::vdisk_free($storecfg, $volid);
5790             }
5791             print STDERR "temporary volume '$volid' sucessfuly removed\n";
5792         };
5793         print STDERR "unable to cleanup '$volid' - $@" if $@;
5794     }
5795 };
5796
5797 sub scan_volids {
5798     my ($cfg, $vmid) = @_;
5799
5800     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5801
5802     my $volid_hash = {};
5803     foreach my $storeid (keys %$info) {
5804         foreach my $item (@{$info->{$storeid}}) {
5805             next if !($item->{volid} && $item->{size});
5806             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5807             $volid_hash->{$item->{volid}} = $item;
5808         }
5809     }
5810
5811     return $volid_hash;
5812 }
5813
5814 sub update_disk_config {
5815     my ($vmid, $conf, $volid_hash) = @_;
5816
5817     my $changes;
5818     my $prefix = "VM $vmid";
5819
5820     # used and unused disks
5821     my $referenced = {};
5822
5823     # Note: it is allowed to define multiple storages with same path (alias), so
5824     # we need to check both 'volid' and real 'path' (two different volid can point
5825     # to the same path).
5826
5827     my $referencedpath = {};
5828
5829     # update size info
5830     PVE::QemuConfig->foreach_volume($conf, sub {
5831         my ($opt, $drive) = @_;
5832
5833         my $volid = $drive->{file};
5834         return if !$volid;
5835
5836         # mark volid as "in-use" for next step
5837         $referenced->{$volid} = 1;
5838         if ($volid_hash->{$volid} &&
5839             (my $path = $volid_hash->{$volid}->{path})) {
5840             $referencedpath->{$path} = 1;
5841         }
5842
5843         return if drive_is_cdrom($drive);
5844         return if !$volid_hash->{$volid};
5845
5846         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volid_hash->{$volid}->{size});
5847         if (defined($updated)) {
5848             $changes = 1;
5849             $conf->{$opt} = print_drive($updated);
5850             print "$prefix ($opt): $msg\n";
5851         }
5852     });
5853
5854     # remove 'unusedX' entry if volume is used
5855     PVE::QemuConfig->foreach_unused_volume($conf, sub {
5856         my ($opt, $drive) = @_;
5857
5858         my $volid = $drive->{file};
5859         return if !$volid;
5860
5861         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5862         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5863             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
5864             $changes = 1;
5865             delete $conf->{$opt};
5866         }
5867
5868         $referenced->{$volid} = 1;
5869         $referencedpath->{$path} = 1 if $path;
5870     });
5871
5872     foreach my $volid (sort keys %$volid_hash) {
5873         next if $volid =~ m/vm-$vmid-state-/;
5874         next if $referenced->{$volid};
5875         my $path = $volid_hash->{$volid}->{path};
5876         next if !$path; # just to be sure
5877         next if $referencedpath->{$path};
5878         $changes = 1;
5879         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
5880         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
5881         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5882     }
5883
5884     return $changes;
5885 }
5886
5887 sub rescan {
5888     my ($vmid, $nolock, $dryrun) = @_;
5889
5890     my $cfg = PVE::Storage::config();
5891
5892     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
5893     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
5894     foreach my $stor (keys %{$cfg->{ids}}) {
5895         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
5896     }
5897
5898     print "rescan volumes...\n";
5899     my $volid_hash = scan_volids($cfg, $vmid);
5900
5901     my $updatefn =  sub {
5902         my ($vmid) = @_;
5903
5904         my $conf = PVE::QemuConfig->load_config($vmid);
5905
5906         PVE::QemuConfig->check_lock($conf);
5907
5908         my $vm_volids = {};
5909         foreach my $volid (keys %$volid_hash) {
5910             my $info = $volid_hash->{$volid};
5911             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5912         }
5913
5914         my $changes = update_disk_config($vmid, $conf, $vm_volids);
5915
5916         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
5917     };
5918
5919     if (defined($vmid)) {
5920         if ($nolock) {
5921             &$updatefn($vmid);
5922         } else {
5923             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5924         }
5925     } else {
5926         my $vmlist = config_list();
5927         foreach my $vmid (keys %$vmlist) {
5928             if ($nolock) {
5929                 &$updatefn($vmid);
5930             } else {
5931                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5932             }
5933         }
5934     }
5935 }
5936
5937 sub restore_proxmox_backup_archive {
5938     my ($archive, $vmid, $user, $options) = @_;
5939
5940     my $storecfg = PVE::Storage::config();
5941
5942     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
5943     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5944
5945     my $server = $scfg->{server};
5946     my $datastore = $scfg->{datastore};
5947     my $username = $scfg->{username} // 'root@pam';
5948     my $fingerprint = $scfg->{fingerprint};
5949     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
5950
5951     my $repo = "$username\@$server:$datastore";
5952
5953     # This is only used for `pbs-restore`!
5954     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
5955     local $ENV{PBS_PASSWORD} = $password;
5956     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
5957
5958     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
5959         PVE::Storage::parse_volname($storecfg, $archive);
5960
5961     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
5962
5963     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
5964
5965     my $tmpdir = "/var/tmp/vzdumptmp$$";
5966     rmtree $tmpdir;
5967     mkpath $tmpdir;
5968
5969     my $conffile = PVE::QemuConfig->config_file($vmid);
5970     my $tmpfn = "$conffile.$$.tmp";
5971      # disable interrupts (always do cleanups)
5972     local $SIG{INT} =
5973         local $SIG{TERM} =
5974         local $SIG{QUIT} =
5975         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
5976
5977     # Note: $oldconf is undef if VM does not exists
5978     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5979     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5980
5981     my $rpcenv = PVE::RPCEnvironment::get();
5982     my $devinfo = {};
5983
5984     eval {
5985         # enable interrupts
5986         local $SIG{INT} =
5987             local $SIG{TERM} =
5988             local $SIG{QUIT} =
5989             local $SIG{HUP} =
5990             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5991
5992         my $cfgfn = "$tmpdir/qemu-server.conf";
5993         my $firewall_config_fn = "$tmpdir/fw.conf";
5994         my $index_fn = "$tmpdir/index.json";
5995
5996         my $cmd = "restore";
5997
5998         my $param = [$pbs_backup_name, "index.json", $index_fn];
5999         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6000         my $index = PVE::Tools::file_get_contents($index_fn);
6001         $index = decode_json($index);
6002
6003         # print Dumper($index);
6004         foreach my $info (@{$index->{files}}) {
6005             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6006                 my $devname = $1;
6007                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6008                     $devinfo->{$devname}->{size} = $1;
6009                 } else {
6010                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6011                 }
6012             }
6013         }
6014
6015         my $is_qemu_server_backup = scalar(grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}});
6016         if (!$is_qemu_server_backup) {
6017             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6018         }
6019         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6020
6021         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6022         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6023
6024         if ($has_firewall_config) {
6025             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6026             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6027
6028             my $pve_firewall_dir = '/etc/pve/firewall';
6029             mkdir $pve_firewall_dir; # make sure the dir exists
6030             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6031         }
6032
6033         my $fh = IO::File->new($cfgfn, "r") ||
6034             die "unable to read qemu-server.conf - $!\n";
6035
6036         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6037
6038         # fixme: rate limit?
6039
6040         # create empty/temp config
6041         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6042
6043         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6044
6045         # allocate volumes
6046         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6047
6048         foreach my $virtdev (sort keys %$virtdev_hash) {
6049             my $d = $virtdev_hash->{$virtdev};
6050             next if $d->{is_cloudinit}; # no need to restore cloudinit
6051
6052             my $volid = $d->{volid};
6053
6054             my $path = PVE::Storage::path($storecfg, $volid);
6055
6056             # This is the ONLY user of the PBS_ env vars set on top of this function!
6057             my $pbs_restore_cmd = [
6058                 '/usr/bin/pbs-restore',
6059                 '--repository', $repo,
6060                 $pbs_backup_name,
6061                 "$d->{devname}.img.fidx",
6062                 $path,
6063                 '--verbose',
6064                 ];
6065
6066             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6067             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
6068
6069             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6070                 push @$pbs_restore_cmd, '--skip-zero';
6071             }
6072
6073             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6074             print "restore proxmox backup image: $dbg_cmdstring\n";
6075             run_command($pbs_restore_cmd);
6076         }
6077
6078         $fh->seek(0, 0) || die "seek failed - $!\n";
6079
6080         my $outfd = new IO::File ($tmpfn, "w") ||
6081             die "unable to write config for VM $vmid\n";
6082
6083         my $cookie = { netcount => 0 };
6084         while (defined(my $line = <$fh>)) {
6085             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $options->{unique});
6086         }
6087
6088         $fh->close();
6089         $outfd->close();
6090     };
6091     my $err = $@;
6092
6093     $restore_deactivate_volumes->($storecfg, $devinfo);
6094
6095     rmtree $tmpdir;
6096
6097     if ($err) {
6098         unlink $tmpfn;
6099         $restore_destroy_volumes->($storecfg, $devinfo);
6100         die $err;
6101     }
6102
6103     rename($tmpfn, $conffile) ||
6104         die "unable to commit configuration file '$conffile'\n";
6105
6106     PVE::Cluster::cfs_update(); # make sure we read new file
6107
6108     eval { rescan($vmid, 1); };
6109     warn $@ if $@;
6110 }
6111
6112 sub restore_vma_archive {
6113     my ($archive, $vmid, $user, $opts, $comp) = @_;
6114
6115     my $readfrom = $archive;
6116
6117     my $cfg = PVE::Storage::config();
6118     my $commands = [];
6119     my $bwlimit = $opts->{bwlimit};
6120
6121     my $dbg_cmdstring = '';
6122     my $add_pipe = sub {
6123         my ($cmd) = @_;
6124         push @$commands, $cmd;
6125         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6126         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6127         $readfrom = '-';
6128     };
6129
6130     my $input = undef;
6131     if ($archive eq '-') {
6132         $input = '<&STDIN';
6133     } else {
6134         # If we use a backup from a PVE defined storage we also consider that
6135         # storage's rate limit:
6136         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6137         if (defined($volid)) {
6138             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6139             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6140             if ($readlimit) {
6141                 print STDERR "applying read rate limit: $readlimit\n";
6142                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6143                 $add_pipe->($cstream);
6144             }
6145         }
6146     }
6147
6148     if ($comp) {
6149         my $info = PVE::Storage::decompressor_info('vma', $comp);
6150         my $cmd = $info->{decompressor};
6151         push @$cmd, $readfrom;
6152         $add_pipe->($cmd);
6153     }
6154
6155     my $tmpdir = "/var/tmp/vzdumptmp$$";
6156     rmtree $tmpdir;
6157
6158     # disable interrupts (always do cleanups)
6159     local $SIG{INT} =
6160         local $SIG{TERM} =
6161         local $SIG{QUIT} =
6162         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6163
6164     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6165     POSIX::mkfifo($mapfifo, 0600);
6166     my $fifofh;
6167
6168     my $openfifo = sub {
6169         open($fifofh, '>', $mapfifo) || die $!;
6170     };
6171
6172     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6173
6174     my $oldtimeout;
6175     my $timeout = 5;
6176
6177     my $devinfo = {};
6178
6179     my $rpcenv = PVE::RPCEnvironment::get();
6180
6181     my $conffile = PVE::QemuConfig->config_file($vmid);
6182     my $tmpfn = "$conffile.$$.tmp";
6183
6184     # Note: $oldconf is undef if VM does not exist
6185     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6186     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6187
6188     my %storage_limits;
6189
6190     my $print_devmap = sub {
6191         my $cfgfn = "$tmpdir/qemu-server.conf";
6192
6193         # we can read the config - that is already extracted
6194         my $fh = IO::File->new($cfgfn, "r") ||
6195             die "unable to read qemu-server.conf - $!\n";
6196
6197         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6198         if (-f $fwcfgfn) {
6199             my $pve_firewall_dir = '/etc/pve/firewall';
6200             mkdir $pve_firewall_dir; # make sure the dir exists
6201             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6202         }
6203
6204         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6205
6206         foreach my $key (keys %storage_limits) {
6207             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6208             next if !$limit;
6209             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6210             $storage_limits{$key} = $limit * 1024;
6211         }
6212
6213         foreach my $devname (keys %$devinfo) {
6214             die "found no device mapping information for device '$devname'\n"
6215                 if !$devinfo->{$devname}->{virtdev};
6216         }
6217
6218         # create empty/temp config
6219         if ($oldconf) {
6220             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6221             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6222         }
6223
6224         # allocate volumes
6225         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6226
6227         # print restore information to $fifofh
6228         foreach my $virtdev (sort keys %$virtdev_hash) {
6229             my $d = $virtdev_hash->{$virtdev};
6230             next if $d->{is_cloudinit}; # no need to restore cloudinit
6231
6232             my $storeid = $d->{storeid};
6233             my $volid = $d->{volid};
6234
6235             my $map_opts = '';
6236             if (my $limit = $storage_limits{$storeid}) {
6237                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6238             }
6239
6240             my $write_zeros = 1;
6241             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6242                 $write_zeros = 0;
6243             }
6244
6245             my $path = PVE::Storage::path($cfg, $volid);
6246
6247             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6248
6249             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6250         }
6251
6252         $fh->seek(0, 0) || die "seek failed - $!\n";
6253
6254         my $outfd = new IO::File ($tmpfn, "w") ||
6255             die "unable to write config for VM $vmid\n";
6256
6257         my $cookie = { netcount => 0 };
6258         while (defined(my $line = <$fh>)) {
6259             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6260         }
6261
6262         $fh->close();
6263         $outfd->close();
6264     };
6265
6266     eval {
6267         # enable interrupts
6268         local $SIG{INT} =
6269             local $SIG{TERM} =
6270             local $SIG{QUIT} =
6271             local $SIG{HUP} =
6272             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6273         local $SIG{ALRM} = sub { die "got timeout\n"; };
6274
6275         $oldtimeout = alarm($timeout);
6276
6277         my $parser = sub {
6278             my $line = shift;
6279
6280             print "$line\n";
6281
6282             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6283                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6284                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6285             } elsif ($line =~ m/^CTIME: /) {
6286                 # we correctly received the vma config, so we can disable
6287                 # the timeout now for disk allocation (set to 10 minutes, so
6288                 # that we always timeout if something goes wrong)
6289                 alarm(600);
6290                 &$print_devmap();
6291                 print $fifofh "done\n";
6292                 my $tmp = $oldtimeout || 0;
6293                 $oldtimeout = undef;
6294                 alarm($tmp);
6295                 close($fifofh);
6296             }
6297         };
6298
6299         print "restore vma archive: $dbg_cmdstring\n";
6300         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6301     };
6302     my $err = $@;
6303
6304     alarm($oldtimeout) if $oldtimeout;
6305
6306     $restore_deactivate_volumes->($cfg, $devinfo);
6307
6308     unlink $mapfifo;
6309     rmtree $tmpdir;
6310
6311     if ($err) {
6312         unlink $tmpfn;
6313         $restore_destroy_volumes->($cfg, $devinfo);
6314         die $err;
6315     }
6316
6317     rename($tmpfn, $conffile) ||
6318         die "unable to commit configuration file '$conffile'\n";
6319
6320     PVE::Cluster::cfs_update(); # make sure we read new file
6321
6322     eval { rescan($vmid, 1); };
6323     warn $@ if $@;
6324 }
6325
6326 sub restore_tar_archive {
6327     my ($archive, $vmid, $user, $opts) = @_;
6328
6329     if ($archive ne '-') {
6330         my $firstfile = tar_archive_read_firstfile($archive);
6331         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6332             if $firstfile ne 'qemu-server.conf';
6333     }
6334
6335     my $storecfg = PVE::Storage::config();
6336
6337     # avoid zombie disks when restoring over an existing VM -> cleanup first
6338     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6339     # skiplock=1 because qmrestore has set the 'create' lock itself already
6340     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6341     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6342
6343     my $tocmd = "/usr/lib/qemu-server/qmextract";
6344
6345     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6346     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6347     $tocmd .= ' --prealloc' if $opts->{prealloc};
6348     $tocmd .= ' --info' if $opts->{info};
6349
6350     # tar option "xf" does not autodetect compression when read from STDIN,
6351     # so we pipe to zcat
6352     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6353         PVE::Tools::shellquote("--to-command=$tocmd");
6354
6355     my $tmpdir = "/var/tmp/vzdumptmp$$";
6356     mkpath $tmpdir;
6357
6358     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6359     local $ENV{VZDUMP_VMID} = $vmid;
6360     local $ENV{VZDUMP_USER} = $user;
6361
6362     my $conffile = PVE::QemuConfig->config_file($vmid);
6363     my $tmpfn = "$conffile.$$.tmp";
6364
6365     # disable interrupts (always do cleanups)
6366     local $SIG{INT} =
6367         local $SIG{TERM} =
6368         local $SIG{QUIT} =
6369         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6370
6371     eval {
6372         # enable interrupts
6373         local $SIG{INT} =
6374             local $SIG{TERM} =
6375             local $SIG{QUIT} =
6376             local $SIG{HUP} =
6377             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6378
6379         if ($archive eq '-') {
6380             print "extracting archive from STDIN\n";
6381             run_command($cmd, input => "<&STDIN");
6382         } else {
6383             print "extracting archive '$archive'\n";
6384             run_command($cmd);
6385         }
6386
6387         return if $opts->{info};
6388
6389         # read new mapping
6390         my $map = {};
6391         my $statfile = "$tmpdir/qmrestore.stat";
6392         if (my $fd = IO::File->new($statfile, "r")) {
6393             while (defined (my $line = <$fd>)) {
6394                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6395                     $map->{$1} = $2 if $1;
6396                 } else {
6397                     print STDERR "unable to parse line in statfile - $line\n";
6398                 }
6399             }
6400             $fd->close();
6401         }
6402
6403         my $confsrc = "$tmpdir/qemu-server.conf";
6404
6405         my $srcfd = new IO::File($confsrc, "r") ||
6406             die "unable to open file '$confsrc'\n";
6407
6408         my $outfd = new IO::File ($tmpfn, "w") ||
6409             die "unable to write config for VM $vmid\n";
6410
6411         my $cookie = { netcount => 0 };
6412         while (defined (my $line = <$srcfd>)) {
6413             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6414         }
6415
6416         $srcfd->close();
6417         $outfd->close();
6418     };
6419     if (my $err = $@) {
6420         unlink $tmpfn;
6421         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6422         die $err;
6423     }
6424
6425     rmtree $tmpdir;
6426
6427     rename $tmpfn, $conffile ||
6428         die "unable to commit configuration file '$conffile'\n";
6429
6430     PVE::Cluster::cfs_update(); # make sure we read new file
6431
6432     eval { rescan($vmid, 1); };
6433     warn $@ if $@;
6434 };
6435
6436 sub foreach_storage_used_by_vm {
6437     my ($conf, $func) = @_;
6438
6439     my $sidhash = {};
6440
6441     PVE::QemuConfig->foreach_volume($conf, sub {
6442         my ($ds, $drive) = @_;
6443         return if drive_is_cdrom($drive);
6444
6445         my $volid = $drive->{file};
6446
6447         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6448         $sidhash->{$sid} = $sid if $sid;
6449     });
6450
6451     foreach my $sid (sort keys %$sidhash) {
6452         &$func($sid);
6453     }
6454 }
6455
6456 my $qemu_snap_storage = {
6457     rbd => 1,
6458 };
6459 sub do_snapshots_with_qemu {
6460     my ($storecfg, $volid) = @_;
6461
6462     my $storage_name = PVE::Storage::parse_volume_id($volid);
6463     my $scfg = $storecfg->{ids}->{$storage_name};
6464
6465     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6466         return 1;
6467     }
6468
6469     if ($volid =~ m/\.(qcow2|qed)$/){
6470         return 1;
6471     }
6472
6473     return undef;
6474 }
6475
6476 sub qga_check_running {
6477     my ($vmid, $nowarn) = @_;
6478
6479     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6480     if ($@) {
6481         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6482         return 0;
6483     }
6484     return 1;
6485 }
6486
6487 sub template_create {
6488     my ($vmid, $conf, $disk) = @_;
6489
6490     my $storecfg = PVE::Storage::config();
6491
6492     PVE::QemuConfig->foreach_volume($conf, sub {
6493         my ($ds, $drive) = @_;
6494
6495         return if drive_is_cdrom($drive);
6496         return if $disk && $ds ne $disk;
6497
6498         my $volid = $drive->{file};
6499         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6500
6501         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6502         $drive->{file} = $voliddst;
6503         $conf->{$ds} = print_drive($drive);
6504         PVE::QemuConfig->write_config($vmid, $conf);
6505     });
6506 }
6507
6508 sub convert_iscsi_path {
6509     my ($path) = @_;
6510
6511     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6512         my $portal = $1;
6513         my $target = $2;
6514         my $lun = $3;
6515
6516         my $initiator_name = get_initiator_name();
6517
6518         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6519                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6520     }
6521
6522     die "cannot convert iscsi path '$path', unkown format\n";
6523 }
6524
6525 sub qemu_img_convert {
6526     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6527
6528     my $storecfg = PVE::Storage::config();
6529     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6530     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6531
6532     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6533
6534     my $cachemode;
6535     my $src_path;
6536     my $src_is_iscsi = 0;
6537     my $src_format;
6538
6539     if ($src_storeid) {
6540         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6541         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6542         $src_format = qemu_img_format($src_scfg, $src_volname);
6543         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6544         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6545         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6546     } elsif (-f $src_volid) {
6547         $src_path = $src_volid;
6548         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6549             $src_format = $1;
6550         }
6551     }
6552
6553     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6554
6555     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6556     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6557     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6558     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6559
6560     my $cmd = [];
6561     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6562     push @$cmd, '-l', "snapshot.name=$snapname"
6563         if $snapname && $src_format && $src_format eq "qcow2";
6564     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6565     push @$cmd, '-T', $cachemode if defined($cachemode);
6566
6567     if ($src_is_iscsi) {
6568         push @$cmd, '--image-opts';
6569         $src_path = convert_iscsi_path($src_path);
6570     } elsif ($src_format) {
6571         push @$cmd, '-f', $src_format;
6572     }
6573
6574     if ($dst_is_iscsi) {
6575         push @$cmd, '--target-image-opts';
6576         $dst_path = convert_iscsi_path($dst_path);
6577     } else {
6578         push @$cmd, '-O', $dst_format;
6579     }
6580
6581     push @$cmd, $src_path;
6582
6583     if (!$dst_is_iscsi && $is_zero_initialized) {
6584         push @$cmd, "zeroinit:$dst_path";
6585     } else {
6586         push @$cmd, $dst_path;
6587     }
6588
6589     my $parser = sub {
6590         my $line = shift;
6591         if($line =~ m/\((\S+)\/100\%\)/){
6592             my $percent = $1;
6593             my $transferred = int($size * $percent / 100);
6594             my $remaining = $size - $transferred;
6595
6596             print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6597         }
6598
6599     };
6600
6601     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6602     my $err = $@;
6603     die "copy failed: $err" if $err;
6604 }
6605
6606 sub qemu_img_format {
6607     my ($scfg, $volname) = @_;
6608
6609     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6610         return $1;
6611     } else {
6612         return "raw";
6613     }
6614 }
6615
6616 sub qemu_drive_mirror {
6617     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
6618
6619     $jobs = {} if !$jobs;
6620
6621     my $qemu_target;
6622     my $format;
6623     $jobs->{"drive-$drive"} = {};
6624
6625     if ($dst_volid =~ /^nbd:/) {
6626         $qemu_target = $dst_volid;
6627         $format = "nbd";
6628     } else {
6629         my $storecfg = PVE::Storage::config();
6630         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6631
6632         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6633
6634         $format = qemu_img_format($dst_scfg, $dst_volname);
6635
6636         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6637
6638         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6639     }
6640
6641     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6642     $opts->{format} = $format if $format;
6643
6644     if (defined($src_bitmap)) {
6645         $opts->{sync} = 'incremental';
6646         $opts->{bitmap} = $src_bitmap;
6647         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
6648     }
6649
6650     if (defined($bwlimit)) {
6651         $opts->{speed} = $bwlimit * 1024;
6652         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6653     } else {
6654         print "drive mirror is starting for drive-$drive\n";
6655     }
6656
6657     # if a job already runs for this device we get an error, catch it for cleanup
6658     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
6659     if (my $err = $@) {
6660         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6661         warn "$@\n" if $@;
6662         die "mirroring error: $err\n";
6663     }
6664
6665     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
6666 }
6667
6668 # $completion can be either
6669 # 'complete': wait until all jobs are ready, block-job-complete them (default)
6670 # 'cancel': wait until all jobs are ready, block-job-cancel them
6671 # 'skip': wait until all jobs are ready, return with block jobs in ready state
6672 sub qemu_drive_mirror_monitor {
6673     my ($vmid, $vmiddst, $jobs, $completion, $qga) = @_;
6674
6675     $completion //= 'complete';
6676
6677     eval {
6678         my $err_complete = 0;
6679
6680         while (1) {
6681             die "storage migration timed out\n" if $err_complete > 300;
6682
6683             my $stats = mon_cmd($vmid, "query-block-jobs");
6684
6685             my $running_mirror_jobs = {};
6686             foreach my $stat (@$stats) {
6687                 next if $stat->{type} ne 'mirror';
6688                 $running_mirror_jobs->{$stat->{device}} = $stat;
6689             }
6690
6691             my $readycounter = 0;
6692
6693             foreach my $job (keys %$jobs) {
6694
6695                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6696                     print "$job : finished\n";
6697                     delete $jobs->{$job};
6698                     next;
6699                 }
6700
6701                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6702
6703                 my $busy = $running_mirror_jobs->{$job}->{busy};
6704                 my $ready = $running_mirror_jobs->{$job}->{ready};
6705                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6706                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6707                     my $remaining = $total - $transferred;
6708                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6709
6710                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6711                 }
6712
6713                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6714             }
6715
6716             last if scalar(keys %$jobs) == 0;
6717
6718             if ($readycounter == scalar(keys %$jobs)) {
6719                 print "all mirroring jobs are ready \n";
6720                 last if $completion eq 'skip'; #do the complete later
6721
6722                 if ($vmiddst && $vmiddst != $vmid) {
6723                     my $agent_running = $qga && qga_check_running($vmid);
6724                     if ($agent_running) {
6725                         print "freeze filesystem\n";
6726                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6727                     } else {
6728                         print "suspend vm\n";
6729                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6730                     }
6731
6732                     # if we clone a disk for a new target vm, we don't switch the disk
6733                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6734
6735                     if ($agent_running) {
6736                         print "unfreeze filesystem\n";
6737                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6738                     } else {
6739                         print "resume vm\n";
6740                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6741                     }
6742
6743                     last;
6744                 } else {
6745
6746                     foreach my $job (keys %$jobs) {
6747                         # try to switch the disk if source and destination are on the same guest
6748                         print "$job: Completing block job...\n";
6749
6750                         my $op;
6751                         if ($completion eq 'complete') {
6752                             $op = 'block-job-complete';
6753                         } elsif ($completion eq 'cancel') {
6754                             $op = 'block-job-cancel';
6755                         } else {
6756                             die "invalid completion value: $completion\n";
6757                         }
6758                         eval { mon_cmd($vmid, $op, device => $job) };
6759                         if ($@ =~ m/cannot be completed/) {
6760                             print "$job: Block job cannot be completed, try again.\n";
6761                             $err_complete++;
6762                         }else {
6763                             print "$job: Completed successfully.\n";
6764                             $jobs->{$job}->{complete} = 1;
6765                         }
6766                     }
6767                 }
6768             }
6769             sleep 1;
6770         }
6771     };
6772     my $err = $@;
6773
6774     if ($err) {
6775         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6776         die "mirroring error: $err";
6777     }
6778
6779 }
6780
6781 sub qemu_blockjobs_cancel {
6782     my ($vmid, $jobs) = @_;
6783
6784     foreach my $job (keys %$jobs) {
6785         print "$job: Cancelling block job\n";
6786         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
6787         $jobs->{$job}->{cancel} = 1;
6788     }
6789
6790     while (1) {
6791         my $stats = mon_cmd($vmid, "query-block-jobs");
6792
6793         my $running_jobs = {};
6794         foreach my $stat (@$stats) {
6795             $running_jobs->{$stat->{device}} = $stat;
6796         }
6797
6798         foreach my $job (keys %$jobs) {
6799
6800             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6801                 print "$job: Done.\n";
6802                 delete $jobs->{$job};
6803             }
6804         }
6805
6806         last if scalar(keys %$jobs) == 0;
6807
6808         sleep 1;
6809     }
6810 }
6811
6812 sub clone_disk {
6813     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6814         $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
6815
6816     my $newvolid;
6817
6818     if (!$full) {
6819         print "create linked clone of drive $drivename ($drive->{file})\n";
6820         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6821         push @$newvollist, $newvolid;
6822     } else {
6823
6824         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6825         $storeid = $storage if $storage;
6826
6827         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6828         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6829
6830         print "create full clone of drive $drivename ($drive->{file})\n";
6831         my $name = undef;
6832         if (drive_is_cloudinit($drive)) {
6833             $name = "vm-$newvmid-cloudinit";
6834             $name .= ".$dst_format" if $dst_format ne 'raw';
6835             $snapname = undef;
6836             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
6837         } elsif ($drivename eq 'efidisk0') {
6838             $size = get_efivars_size($conf);
6839         }
6840         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6841         push @$newvollist, $newvolid;
6842
6843         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6844
6845         if (drive_is_cloudinit($drive)) {
6846             goto no_data_clone;
6847         }
6848
6849         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6850         if (!$running || $snapname) {
6851             # TODO: handle bwlimits
6852             if ($drivename eq 'efidisk0') {
6853                 # the relevant data on the efidisk may be smaller than the source
6854                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
6855                 # that is given by the OVMF_VARS.fd
6856                 my $src_path = PVE::Storage::path($storecfg, $drive->{file});
6857                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
6858                 run_command(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=1", "count=$size", "if=$src_path", "of=$dst_path"]);
6859             } else {
6860                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6861             }
6862         } else {
6863
6864             my $kvmver = get_running_qemu_version ($vmid);
6865             if (!min_version($kvmver, 2, 7)) {
6866                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6867                     if $drive->{iothread};
6868             }
6869
6870             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $completion, $qga, $bwlimit);
6871         }
6872     }
6873
6874 no_data_clone:
6875     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6876
6877     my $disk = $drive;
6878     $disk->{format} = undef;
6879     $disk->{file} = $newvolid;
6880     $disk->{size} = $size;
6881
6882     return $disk;
6883 }
6884
6885 sub get_running_qemu_version {
6886     my ($vmid) = @_;
6887     my $res = mon_cmd($vmid, "query-version");
6888     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6889 }
6890
6891 sub qemu_use_old_bios_files {
6892     my ($machine_type) = @_;
6893
6894     return if !$machine_type;
6895
6896     my $use_old_bios_files = undef;
6897
6898     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6899         $machine_type = $1;
6900         $use_old_bios_files = 1;
6901     } else {
6902         my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
6903         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6904         # load new efi bios files on migration. So this hack is required to allow
6905         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6906         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6907         $use_old_bios_files = !min_version($version, 2, 4);
6908     }
6909
6910     return ($use_old_bios_files, $machine_type);
6911 }
6912
6913 sub get_efivars_size {
6914     my ($conf) = @_;
6915     my $arch = get_vm_arch($conf);
6916     my (undef, $ovmf_vars) = get_ovmf_files($arch);
6917     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
6918     return -s $ovmf_vars;
6919 }
6920
6921 sub update_efidisk_size {
6922     my ($conf) = @_;
6923
6924     return if !defined($conf->{efidisk0});
6925
6926     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
6927     $disk->{size} = get_efivars_size($conf);
6928     $conf->{efidisk0} = print_drive($disk);
6929
6930     return;
6931 }
6932
6933 sub create_efidisk($$$$$) {
6934     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
6935
6936     my (undef, $ovmf_vars) = get_ovmf_files($arch);
6937     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
6938
6939     my $vars_size_b = -s $ovmf_vars;
6940     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
6941     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6942     PVE::Storage::activate_volumes($storecfg, [$volid]);
6943
6944     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
6945     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
6946
6947     return ($volid, $size/1024);
6948 }
6949
6950 sub vm_iothreads_list {
6951     my ($vmid) = @_;
6952
6953     my $res = mon_cmd($vmid, 'query-iothreads');
6954
6955     my $iothreads = {};
6956     foreach my $iothread (@$res) {
6957         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6958     }
6959
6960     return $iothreads;
6961 }
6962
6963 sub scsihw_infos {
6964     my ($conf, $drive) = @_;
6965
6966     my $maxdev = 0;
6967
6968     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6969         $maxdev = 7;
6970     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6971         $maxdev = 1;
6972     } else {
6973         $maxdev = 256;
6974     }
6975
6976     my $controller = int($drive->{index} / $maxdev);
6977     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6978
6979     return ($maxdev, $controller, $controller_prefix);
6980 }
6981
6982 sub windows_version {
6983     my ($ostype) = @_;
6984
6985     return 0 if !$ostype;
6986
6987     my $winversion = 0;
6988
6989     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6990         $winversion = 5;
6991     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6992         $winversion = 6;
6993     } elsif ($ostype =~ m/^win(\d+)$/) {
6994         $winversion = $1;
6995     }
6996
6997     return $winversion;
6998 }
6999
7000 sub resolve_dst_disk_format {
7001         my ($storecfg, $storeid, $src_volname, $format) = @_;
7002         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7003
7004         if (!$format) {
7005             # if no target format is specified, use the source disk format as hint
7006             if ($src_volname) {
7007                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7008                 $format = qemu_img_format($scfg, $src_volname);
7009             } else {
7010                 return $defFormat;
7011             }
7012         }
7013
7014         # test if requested format is supported - else use default
7015         my $supported = grep { $_ eq $format } @$validFormats;
7016         $format = $defFormat if !$supported;
7017         return $format;
7018 }
7019
7020 # NOTE: if this logic changes, please update docs & possibly gui logic
7021 sub find_vmstate_storage {
7022     my ($conf, $storecfg) = @_;
7023
7024     # first, return storage from conf if set
7025     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7026
7027     my ($target, $shared, $local);
7028
7029     foreach_storage_used_by_vm($conf, sub {
7030         my ($sid) = @_;
7031         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7032         my $dst = $scfg->{shared} ? \$shared : \$local;
7033         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7034     });
7035
7036     # second, use shared storage where VM has at least one disk
7037     # third, use local storage where VM has at least one disk
7038     # fall back to local storage
7039     $target = $shared // $local // 'local';
7040
7041     return $target;
7042 }
7043
7044 sub generate_uuid {
7045     my ($uuid, $uuid_str);
7046     UUID::generate($uuid);
7047     UUID::unparse($uuid, $uuid_str);
7048     return $uuid_str;
7049 }
7050
7051 sub generate_smbios1_uuid {
7052     return "uuid=".generate_uuid();
7053 }
7054
7055 sub nbd_stop {
7056     my ($vmid) = @_;
7057
7058     mon_cmd($vmid, 'nbd-server-stop');
7059 }
7060
7061 sub create_reboot_request {
7062     my ($vmid) = @_;
7063     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7064         or die "failed to create reboot trigger file: $!\n";
7065     close($fh);
7066 }
7067
7068 sub clear_reboot_request {
7069     my ($vmid) = @_;
7070     my $path = "/run/qemu-server/$vmid.reboot";
7071     my $res = 0;
7072
7073     $res = unlink($path);
7074     die "could not remove reboot request for $vmid: $!"
7075         if !$res && $! != POSIX::ENOENT;
7076
7077     return $res;
7078 }
7079
7080 # bash completion helper
7081
7082 sub complete_backup_archives {
7083     my ($cmdname, $pname, $cvalue) = @_;
7084
7085     my $cfg = PVE::Storage::config();
7086
7087     my $storeid;
7088
7089     if ($cvalue =~ m/^([^:]+):/) {
7090         $storeid = $1;
7091     }
7092
7093     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7094
7095     my $res = [];
7096     foreach my $id (keys %$data) {
7097         foreach my $item (@{$data->{$id}}) {
7098             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
7099             push @$res, $item->{volid} if defined($item->{volid});
7100         }
7101     }
7102
7103     return $res;
7104 }
7105
7106 my $complete_vmid_full = sub {
7107     my ($running) = @_;
7108
7109     my $idlist = vmstatus();
7110
7111     my $res = [];
7112
7113     foreach my $id (keys %$idlist) {
7114         my $d = $idlist->{$id};
7115         if (defined($running)) {
7116             next if $d->{template};
7117             next if $running && $d->{status} ne 'running';
7118             next if !$running && $d->{status} eq 'running';
7119         }
7120         push @$res, $id;
7121
7122     }
7123     return $res;
7124 };
7125
7126 sub complete_vmid {
7127     return &$complete_vmid_full();
7128 }
7129
7130 sub complete_vmid_stopped {
7131     return &$complete_vmid_full(0);
7132 }
7133
7134 sub complete_vmid_running {
7135     return &$complete_vmid_full(1);
7136 }
7137
7138 sub complete_storage {
7139
7140     my $cfg = PVE::Storage::config();
7141     my $ids = $cfg->{ids};
7142
7143     my $res = [];
7144     foreach my $sid (keys %$ids) {
7145         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7146         next if !$ids->{$sid}->{content}->{images};
7147         push @$res, $sid;
7148     }
7149
7150     return $res;
7151 }
7152
7153 sub complete_migration_storage {
7154     my ($cmd, $param, $current_value, $all_args) = @_;
7155
7156     my $targetnode = @$all_args[1];
7157
7158     my $cfg = PVE::Storage::config();
7159     my $ids = $cfg->{ids};
7160
7161     my $res = [];
7162     foreach my $sid (keys %$ids) {
7163         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7164         next if !$ids->{$sid}->{content}->{images};
7165         push @$res, $sid;
7166     }
7167
7168     return $res;
7169 }
7170
7171 1;